lib/isatty.c

   1 /* isatty() replacement.
   2    Copyright (C) 2012-2020 Free Software Foundation, Inc.
   3
   4    This program is free software: you can redistribute it and/or modify
   5    it under the terms of the GNU General Public License as published by
   6    the Free Software Foundation; either version 3 of the License, or
   7    (at your option) any later version.
   8
   9    This program is distributed in the hope that it will be useful,
  10    but WITHOUT ANY WARRANTY; without even the implied warranty of
  11    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12    GNU General Public License for more details.
  13
  14    You should have received a copy of the GNU General Public License
  15    along with this program.  If not, see <https://www.gnu.org/licenses/>.  */
  16
  17 #include <config.h>
  18
  19 /* Specification.  */
  20 #include <unistd.h>
  21
  22 /* This replacement is enabled on native Windows.  */
  23
  24 #include <errno.h>
  25 #include <string.h>
  26
  27 /* Get declarations of the Win32 API functions.  */
  28 #define WIN32_LEAN_AND_MEAN
  29 #include <windows.h>
  30
  31 #if HAVE_MSVC_INVALID_PARAMETER_HANDLER
  32 # include "msvc-inval.h"
  33 #endif
  34
  35 /* Get _get_osfhandle().  */
  36 #if GNULIB_MSVC_NOTHROW
  37 # include "msvc-nothrow.h"
  38 #else
  39 # include <io.h>
  40 #endif
  41
  42 /* Avoid warnings from gcc -Wcast-function-type.  */
  43 #define GetProcAddress \
  44   (void *) GetProcAddress
  45
  46 /* GetNamedPipeClientProcessId was introduced only in Windows Vista.  */
  47 typedef BOOL (WINAPI * GetNamedPipeClientProcessIdFuncType) (HANDLE hPipe,
  48                                                              PULONG pClientProcessId);
  49 static GetNamedPipeClientProcessIdFuncType GetNamedPipeClientProcessIdFunc = NULL;
  50 /* QueryFullProcessImageName was introduced only in Windows Vista.  */
  51 typedef BOOL (WINAPI * QueryFullProcessImageNameFuncType) (HANDLE hProcess,
  52                                                            DWORD dwFlags,
  53                                                            LPSTR lpExeName,
  54                                                            PDWORD pdwSize);
  55 static QueryFullProcessImageNameFuncType QueryFullProcessImageNameFunc = NULL;
  56 static BOOL initialized = FALSE;
  57
  58 static void
  59 initialize (void)
  60 {
  61   HMODULE kernel32 = LoadLibrary ("kernel32.dll");
  62   if (kernel32 != NULL)
  63     {
  64       GetNamedPipeClientProcessIdFunc =
  65         (GetNamedPipeClientProcessIdFuncType) GetProcAddress (kernel32, "GetNamedPipeClientProcessId");
  66       QueryFullProcessImageNameFunc =
  67         (QueryFullProcessImageNameFuncType) GetProcAddress (kernel32, "QueryFullProcessImageNameA");
  68     }
  69   initialized = TRUE;
  70 }
  71
  72 static BOOL IsConsoleHandle (HANDLE h)
  73 {
  74   DWORD mode;
  75   /* GetConsoleMode
  76      <https://docs.microsoft.com/en-us/windows/console/getconsolemode> */
  77   return GetConsoleMode (h, &mode) != 0;
  78 }
  79
  80 static BOOL IsCygwinConsoleHandle (HANDLE h)
  81 {
  82   /* A handle to a Cygwin console is in fact a named pipe whose client process
  83      and server process is <CYGWIN_INSTALL_DIR>\bin\mintty.exe.  */
  84   BOOL result = FALSE;
  85   ULONG processId;
  86
  87   if (!initialized)
  88     initialize ();
  89
  90   /* GetNamedPipeClientProcessId
  91      <https://docs.microsoft.com/en-us/windows/desktop/api/winbase/nf-winbase-getnamedpipeclientprocessid>
  92      It requires -D_WIN32_WINNT=_WIN32_WINNT_VISTA or higher.  */
  93   if (GetNamedPipeClientProcessIdFunc && QueryFullProcessImageNameFunc
  94       && GetNamedPipeClientProcessIdFunc (h, &processId))
  95     {
  96       /* OpenProcess
  97          <https://docs.microsoft.com/en-us/windows/desktop/api/processthreadsapi/nf-processthreadsapi-openprocess> */
  98       HANDLE processHandle =
  99         OpenProcess (PROCESS_QUERY_LIMITED_INFORMATION, FALSE, processId);
 100       if (processHandle != NULL)
 101         {
 102           char buf[1024];
 103           DWORD bufsize = sizeof (buf);
 104           /* The file name can be determined through
 105              GetProcessImageFileName
 106              <https://docs.microsoft.com/en-us/windows/desktop/api/psapi/nf-psapi-getprocessimagefilenamea>
 107              or
 108              QueryFullProcessImageName
 109              <https://docs.microsoft.com/en-us/windows/desktop/api/winbase/nf-winbase-queryfullprocessimagenamea>
 110              The former returns a file name in non-standard notation (it starts
 111              with '\Device\') and may require linking with psapi.dll.
 112              The latter is better, but requires -D_WIN32_WINNT=_WIN32_WINNT_VISTA
 113              or higher.  */
 114           if (QueryFullProcessImageNameFunc (processHandle, 0, buf, &bufsize))
 115             {
 116               if (strlen (buf) >= 11
 117                   && strcmp (buf + strlen (buf) - 11, "\\mintty.exe") == 0)
 118                 result = TRUE;
 119             }
 120           CloseHandle (processHandle);
 121         }
 122     }
 123   return result;
 124 }
 125
 126 #if HAVE_MSVC_INVALID_PARAMETER_HANDLER
 127 static int
 128 _isatty_nothrow (int fd)
 129 {
 130   int result;
 131
 132   TRY_MSVC_INVAL
 133     {
 134       result = _isatty (fd);
 135     }
 136   CATCH_MSVC_INVAL
 137     {
 138       result = 0;
 139     }
 140   DONE_MSVC_INVAL;
 141
 142   return result;
 143 }
 144 #else
 145 # define _isatty_nothrow _isatty
 146 #endif
 147
 148 /* Determine whether FD refers to a console device.  Return 1 if yes.
 149    Return 0 and set errno if no. (ptsname_r relies on the errno value.)  */
 150 int
 151 isatty (int fd)
 152 {
 153   HANDLE h = (HANDLE) _get_osfhandle (fd);
 154   if (h == INVALID_HANDLE_VALUE)
 155     {
 156       errno = EBADF;
 157       return 0;
 158     }
 159   /* _isatty (fd) tests whether GetFileType of the handle is FILE_TYPE_CHAR.
 160      But it does not set errno when it returns 0.  */
 161   if (_isatty_nothrow (fd))
 162     {
 163       if (IsConsoleHandle (h))
 164         return 1;
 165     }
 166   if (IsCygwinConsoleHandle (h))
 167     return 1;
 168   errno = ENOTTY;
 169   return 0;
 170 }