xapian-core/tests/harness/backendmanager_remotetcp.cc

   1 /** @file backendmanager_remotetcp.cc
   2  * @brief BackendManager subclass for remotetcp databases.
   3  */
   4 /* Copyright (C) 2006,2007,2008,2009,2013,2015 Olly Betts
   5  * Copyright (C) 2008 Lemur Consulting Ltd
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License as
   9  * published by the Free Software Foundation; either version 2 of the
  10  * License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  20  */
  21
  22 #include <config.h>
  23
  24 #include "backendmanager_remotetcp.h"
  25
  26 #include <xapian.h>
  27
  28 #include <stdio.h> // For fdopen().
  29 #include <cerrno>
  30 #include <cstring>
  31
  32 #ifdef HAVE_FORK
  33 # include <signal.h>
  34 # include <sys/types.h>
  35 # include "safesyssocket.h"
  36 # include <sys/wait.h>
  37 # include <unistd.h>
  38 // Some older systems had SIGCLD rather than SIGCHLD.
  39 # if !defined SIGCHLD && defined SIGCLD
  40 #  define SIGCHLD SIGCLD
  41 # endif
  42 #endif
  43
  44 #ifdef __WIN32__
  45 # include <io.h> // For _open_osfhandle().
  46 # include "safefcntl.h"
  47 # include "safewindows.h"
  48 # include <cstdlib> // For free().
  49 #endif
  50
  51 #include "str.h"
  52
  53 #include <string>
  54 #include <vector>
  55
  56 #ifdef HAVE_VALGRIND
  57 # include <valgrind/memcheck.h>
  58 #endif
  59
  60 using namespace std;
  61
  62 // We've had problems on some hosts which run tinderbox tests with "localhost"
  63 // not being set in /etc/hosts - using the IP address equivalent seems more
  64 // reliable.
  65 #define LOCALHOST "127.0.0.1"
  66
  67 // Start at DEFAULT port and try higher ports until one isn't already in use.
  68 #define DEFAULT_PORT 1239
  69
  70 #ifdef HAVE_FORK
  71
  72 // We can't dynamically allocate memory for this because it confuses the leak
  73 // detector.  We only have 1-3 child fds open at once anyway, so a fixed size
  74 // array isn't a problem, and linear scanning isn't a problem either.
  75 struct pid_fd {
  76     pid_t pid;
  77     int fd;
  78 };
  79
  80 static pid_fd pid_to_fd[16];
  81
  82 extern "C" {
  83
  84 static void
  85 on_SIGCHLD(int /*sig*/)
  86 {
  87     int status;
  88     pid_t child;
  89     while ((child = waitpid(-1, &status, WNOHANG)) > 0) {
  90         for (unsigned i = 0; i < sizeof(pid_to_fd) / sizeof(pid_fd); ++i) {
  91             if (pid_to_fd[i].pid == child) {
  92                 int fd = pid_to_fd[i].fd;
  93                 pid_to_fd[i].fd = 0;
  94                 pid_to_fd[i].pid = 0;
  95                 // NB close() *is* safe to use in a signal handler.
  96                 close(fd);
  97                 break;
  98             }
  99         }
 100     }
 101 }
 102
 103 }
 104
 105 static int
 106 launch_xapian_tcpsrv(const string & args)
 107 {
 108     int port = DEFAULT_PORT;
 109
 110     // We want to be able to get the exit status of the child process we fork
 111     // if xapian-tcpsrv doesn't start listening successfully.
 112     signal(SIGCHLD, SIG_DFL);
 113 try_next_port:
 114     string cmd = XAPIAN_TCPSRV " --one-shot --interface " LOCALHOST " --port ";
 115     cmd += str(port);
 116     cmd += " ";
 117     cmd += args;
 118 #ifdef HAVE_VALGRIND
 119     if (RUNNING_ON_VALGRIND) cmd = "./runsrv " + cmd;
 120 #endif
 121     int fds[2];
 122     if (socketpair(AF_UNIX, SOCK_STREAM|SOCK_CLOEXEC, PF_UNSPEC, fds) < 0) {
 123         string msg("Couldn't create socketpair: ");
 124         msg += strerror(errno);
 125         throw msg;
 126     }
 127
 128     pid_t child = fork();
 129     if (child == 0) {
 130         // Child process.
 131         close(fds[0]);
 132         // Connect stdout and stderr to the socket.
 133         //
 134         // Make sure the socket isn't fd 1 or 2.  We need to ensure that
 135         // FD_CLOEXEC isn't set for stdout or stderr (which creating them with
 136         // dup2() achieves), and that we close fds[1].  The cleanest way to
 137         // address this seems to be to turn the unusual situation into the
 138         // usual one.
 139         if (fds[1] == 1 || fds[1] == 2) {
 140             dup2(fds[1], 3);
 141             fds[1] = 3;
 142         }
 143         dup2(fds[1], 1);
 144         dup2(fds[1], 2);
 145         close(fds[1]);
 146         execl("/bin/sh", "/bin/sh", "-c", cmd.c_str(), static_cast<void*>(0));
 147         _exit(-1);
 148     }
 149
 150     close(fds[1]);
 151     if (child == -1) {
 152         // Couldn't fork.
 153         int fork_errno = errno;
 154         close(fds[0]);
 155         string msg("Couldn't fork: ");
 156         msg += strerror(fork_errno);
 157         throw msg;
 158     }
 159
 160     // Parent process.
 161
 162     // Wrap the file descriptor in a FILE * so we can read lines using fgets().
 163     FILE * fh = fdopen(fds[0], "r");
 164     if (fh == NULL) {
 165         string msg("Failed to run command '");
 166         msg += cmd;
 167         msg += "': ";
 168         msg += strerror(errno);
 169         throw msg;
 170     }
 171
 172     string output;
 173     while (true) {
 174         char buf[256];
 175         if (fgets(buf, sizeof(buf), fh) == NULL) {
 176             fclose(fh);
 177             // Wait for the child to exit.
 178             int status;
 179             if (waitpid(child, &status, 0) == -1) {
 180                 string msg("waitpid failed: ");
 181                 msg += strerror(errno);
 182                 throw msg;
 183             }
 184             if (++port < 65536 && status != 0) {
 185                 if (WIFEXITED(status) && WEXITSTATUS(status) == 69) {
 186                     // 69 is EX_UNAVAILABLE which xapian-tcpsrv exits
 187                     // with if (and only if) the port specified was
 188                     // in use.
 189                     goto try_next_port;
 190                 }
 191             }
 192             string msg("Failed to get 'Listening...' from command '");
 193             msg += cmd;
 194             msg += "' (output: ";
 195             msg += output;
 196             msg += ")";
 197             throw msg;
 198         }
 199         if (strcmp(buf, "Listening...\n") == 0) break;
 200         output += buf;
 201     }
 202
 203     // dup() the fd we wrapped with fdopen() so we can keep it open so the
 204     // xapian-tcpsrv keeps running.
 205     int tracked_fd = dup(fds[0]);
 206
 207     // We must fclose() the FILE* to avoid valgrind detecting memory leaks from
 208     // its buffers.
 209     fclose(fh);
 210
 211     // Find a slot to track the pid->fd mapping in.  If we can't find a slot
 212     // it just means we'll leak the fd, so don't worry about that too much.
 213     for (unsigned i = 0; i < sizeof(pid_to_fd) / sizeof(pid_fd); ++i) {
 214         if (pid_to_fd[i].pid == 0) {
 215             pid_to_fd[i].fd = tracked_fd;
 216             pid_to_fd[i].pid = child;
 217             break;
 218         }
 219     }
 220
 221     // Set a signal handler to clean up the xapian-tcpsrv child process when it
 222     // finally exits.
 223     signal(SIGCHLD, on_SIGCHLD);
 224
 225     return port;
 226 }
 227
 228 #elif defined __WIN32__
 229
 230 [[noreturn]]
 231 static void win32_throw_error_string(const char * str)
 232 {
 233     string msg(str);
 234     char * error = 0;
 235     DWORD len;
 236     len = FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM|FORMAT_MESSAGE_ALLOCATE_BUFFER,
 237                         0, GetLastError(), 0, (CHAR*)&error, 0, 0);
 238     if (error) {
 239         // Remove any trailing \r\n from output of FormatMessage.
 240         if (len >= 2 && error[len - 2] == '\r' && error[len - 1] == '\n')
 241             len -= 2;
 242         if (len) {
 243             msg += ": ";
 244             msg.append(error, len);
 245         }
 246         LocalFree(error);
 247     }
 248     throw msg;
 249 }
 250
 251 // This implementation uses the WIN32 API to start xapian-tcpsrv as a child
 252 // process and read its output using a pipe.
 253 static int
 254 launch_xapian_tcpsrv(const string & args)
 255 {
 256     int port = DEFAULT_PORT;
 257
 258 try_next_port:
 259     string cmd = XAPIAN_TCPSRV " --one-shot --interface " LOCALHOST " --port ";
 260     cmd += str(port);
 261     cmd += " ";
 262     cmd += args;
 263
 264     // Create a pipe so we can read stdout/stderr from the child process.
 265     HANDLE hRead, hWrite;
 266     if (!CreatePipe(&hRead, &hWrite, 0, 0))
 267         win32_throw_error_string("Couldn't create pipe");
 268
 269     // Set the write handle to be inherited by the child process.
 270     SetHandleInformation(hWrite, HANDLE_FLAG_INHERIT, 1);
 271
 272     // Create the child process.
 273     PROCESS_INFORMATION procinfo;
 274     memset(&procinfo, 0, sizeof(PROCESS_INFORMATION));
 275
 276     STARTUPINFO startupinfo;
 277     memset(&startupinfo, 0, sizeof(STARTUPINFO));
 278     startupinfo.cb = sizeof(STARTUPINFO);
 279     startupinfo.hStdError = hWrite;
 280     startupinfo.hStdOutput = hWrite;
 281     startupinfo.hStdInput = INVALID_HANDLE_VALUE;
 282     startupinfo.dwFlags |= STARTF_USESTDHANDLES;
 283
 284     // For some reason Windows wants a modifiable copy!
 285     BOOL ok;
 286     char * cmdline = strdup(cmd.c_str());
 287     ok = CreateProcess(0, cmdline, 0, 0, TRUE, 0, 0, 0, &startupinfo, &procinfo);
 288     free(cmdline);
 289     if (!ok)
 290         win32_throw_error_string("Couldn't create child process");
 291
 292     CloseHandle(hWrite);
 293     CloseHandle(procinfo.hThread);
 294
 295     string output;
 296     FILE *fh = fdopen(_open_osfhandle(intptr_t(hRead), O_RDONLY), "r");
 297     while (true) {
 298         char buf[256];
 299         if (fgets(buf, sizeof(buf), fh) == NULL) {
 300             fclose(fh);
 301             DWORD rc;
 302             // This doesn't seem to be necessary on the machine I tested on,
 303             // but I guess it could be on a slow machine...
 304             while (GetExitCodeProcess(procinfo.hProcess, &rc) && rc == STILL_ACTIVE) {
 305                 Sleep(100);
 306             }
 307             CloseHandle(procinfo.hProcess);
 308             if (++port < 65536 && rc == 69) {
 309                 // 69 is EX_UNAVAILABLE which xapian-tcpsrv exits
 310                 // with if (and only if) the port specified was
 311                 // in use.
 312                 goto try_next_port;
 313             }
 314             string msg("Failed to get 'Listening...' from command '");
 315             msg += cmd;
 316             msg += "' (output: ";
 317             msg += output;
 318             msg += ")";
 319             throw msg;
 320         }
 321         if (strcmp(buf, "Listening...\r\n") == 0) break;
 322         output += buf;
 323     }
 324     fclose(fh);
 325
 326     return port;
 327 }
 328
 329 #else
 330 # error Neither HAVE_FORK nor __WIN32__ is defined
 331 #endif
 332
 333 BackendManagerRemoteTcp::~BackendManagerRemoteTcp() {
 334     BackendManagerRemoteTcp::clean_up();
 335 }
 336
 337 std::string
 338 BackendManagerRemoteTcp::get_dbtype() const
 339 {
 340     return "remotetcp_" + sub_manager->get_dbtype();
 341 }
 342
 343 Xapian::Database
 344 BackendManagerRemoteTcp::do_get_database(const vector<string> & files)
 345 {
 346     // Default to a long (5 minute) timeout so that tests won't fail just
 347     // because the host is slow or busy.
 348     return BackendManagerRemoteTcp::get_remote_database(files, 300000);
 349 }
 350
 351 Xapian::WritableDatabase
 352 BackendManagerRemoteTcp::get_writable_database(const string & name,
 353                                                const string & file)
 354 {
 355     string args = get_writable_database_args(name, file);
 356     int port = launch_xapian_tcpsrv(args);
 357     return Xapian::Remote::open_writable(LOCALHOST, port);
 358 }
 359
 360 Xapian::Database
 361 BackendManagerRemoteTcp::get_remote_database(const vector<string> & files,
 362                                              unsigned int timeout)
 363 {
 364     string args = get_remote_database_args(files, timeout);
 365     int port = launch_xapian_tcpsrv(args);
 366     return Xapian::Remote::open(LOCALHOST, port);
 367 }
 368
 369 Xapian::Database
 370 BackendManagerRemoteTcp::get_writable_database_as_database()
 371 {
 372     string args = get_writable_database_as_database_args();
 373     int port = launch_xapian_tcpsrv(args);
 374     return Xapian::Remote::open(LOCALHOST, port);
 375 }
 376
 377 Xapian::WritableDatabase
 378 BackendManagerRemoteTcp::get_writable_database_again()
 379 {
 380     string args = get_writable_database_again_args();
 381     int port = launch_xapian_tcpsrv(args);
 382     return Xapian::Remote::open_writable(LOCALHOST, port);
 383 }
 384
 385 void
 386 BackendManagerRemoteTcp::clean_up()
 387 {
 388 #ifdef HAVE_FORK
 389     signal(SIGCHLD, SIG_DFL);
 390     for (unsigned i = 0; i < sizeof(pid_to_fd) / sizeof(pid_fd); ++i) {
 391         pid_t child = pid_to_fd[i].pid;
 392         if (child) {
 393             int status;
 394             while (waitpid(child, &status, 0) == -1 && errno == EINTR) { }
 395             // Other possible error from waitpid is ECHILD, which it seems can
 396             // only mean that the child has already exited and SIGCHLD was set
 397             // to SIG_IGN.  If we did somehow see that, the sanest response
 398             // seems to be to close the fd and move on.
 399             int fd = pid_to_fd[i].fd;
 400             pid_to_fd[i].fd = 0;
 401             pid_to_fd[i].pid = 0;
 402             close(fd);
 403         }
 404     }
 405 #endif
 406 }