[NTVDM]

Do not rely on the internal format for console handles. Instead, use a more reliable way, by checking for success when calling a simple console function acting on in/out console handles.

svn path=/branches/ntvdm/; revision=59746

subsystems/ntvdm/dos.c

@@ -29,6 +29,30 @@ static BOOLEAN DosUmbLinked = FALSE;
 
 /* PRIVATE FUNCTIONS **********************************************************/
 
+/* Taken from base/shell/cmd/console.c */
+static BOOL IsConsoleHandle(HANDLE hHandle)
+{
+    DWORD dwMode;
+
+    /* Check whether the handle may be that of a console... */
+    if ((GetFileType(hHandle) & FILE_TYPE_CHAR) == 0) return FALSE;
+
+    /*
+     * It may be. Perform another test... The idea comes from the
+     * MSDN description of the WriteConsole API:
+     *
+     * "WriteConsole fails if it is used with a standard handle
+     *  that is redirected to a file. If an application processes
+     *  multilingual output that can be redirected, determine whether
+     *  the output handle is a console handle (one method is to call
+     *  the GetConsoleMode function and check whether it succeeds).
+     *  If the handle is a console handle, call WriteConsole. If the
+     *  handle is not a console handle, the output is redirected and
+     *  you should call WriteFile to perform the I/O."
+     */
+    return GetConsoleMode(hHandle, &dwMode);
+}
+
 static VOID DosCombineFreeBlocks(WORD StartBlock)
 {
     PDOS_MCB CurrentMcb = SEGMENT_TO_MCB(StartBlock), NextMcb;

subsystems/ntvdm/ntvdm.h

@@ -26,7 +26,6 @@
 #define MAX_ADDRESS TO_LINEAR(MAX_SEGMENT, MAX_OFFSET)
 #define FAR_POINTER(x) ((ULONG_PTR)BaseAddress + TO_LINEAR(HIWORD(x), LOWORD(x)))
 #define STEPS_PER_CYCLE 256
-#define IsConsoleHandle(h) (((((ULONG_PTR)h) & 0x10000003) == 3) ? TRUE : FALSE)
 
 /* FUNCTIONS ******************************************************************/