contrib/pg_standby/pg_standby.c

   1 /*
   2  * pg_standby.c
   3  *
   4  * Production-ready example of how to create a Warm Standby
   5  * database server using continuous archiving as a
   6  * replication mechanism
   7  *
   8  * We separate the parameters for archive and nextWALfile
   9  * so that we can check the archive exists, even if the
  10  * WAL file doesn't (yet).
  11  *
  12  * This program will be executed once in full for each file
  13  * requested by the warm standby server.
  14  *
  15  * It is designed to cater to a variety of needs, as well
  16  * providing a customizable section.
  17  *
  18  * Original author:             Simon Riggs  simon@2ndquadrant.com
  19  * Current maintainer:  Simon Riggs
  20  */
  21 #include "postgres_fe.h"
  22
  23 #include <ctype.h>
  24 #include <dirent.h>
  25 #include <sys/stat.h>
  26 #include <signal.h>
  27
  28 #ifdef WIN32
  29 int                     getopt(int argc, char *const argv[], const char *optstring);
  30 #else
  31 #include <sys/time.h>
  32 #include <unistd.h>
  33
  34 #ifdef HAVE_GETOPT_H
  35 #include <getopt.h>
  36 #endif
  37 #endif   /* ! WIN32 */
  38
  39 extern char *optarg;
  40 extern int      optind;
  41
  42 /* Options and defaults */
  43 int                     sleeptime = 5;          /* amount of time to sleep between file checks */
  44 int                     waittime = -1;          /* how long we have been waiting, -1 no wait
  45                                                                  * yet */
  46 int                     maxwaittime = 0;        /* how long are we prepared to wait for? */
  47 int                     keepfiles = 0;          /* number of WAL files to keep, 0 keep all */
  48 int                     maxretries = 3;         /* number of retries on restore command */
  49 bool            debug = false;          /* are we debugging? */
  50 bool            triggered = false;      /* have we been triggered? */
  51 bool            need_cleanup = false;           /* do we need to remove files from
  52                                                                                  * archive? */
  53
  54 static volatile sig_atomic_t signaled = false;
  55
  56 char       *archiveLocation;    /* where to find the archive? */
  57 char       *triggerPath;                /* where to find the trigger file? */
  58 char       *xlogFilePath;               /* where we are going to restore to */
  59 char       *nextWALFileName;    /* the file we need to get from archive */
  60 char       *restartWALFileName; /* the file from which we can restart restore */
  61 char       *priorWALFileName;   /* the file we need to get from archive */
  62 char            WALFilePath[MAXPGPATH];         /* the file path including archive */
  63 char            restoreCommand[MAXPGPATH];      /* run this to restore */
  64 char            exclusiveCleanupFileName[MAXPGPATH];            /* the file we need to
  65                                                                                                                  * get from archive */
  66
  67 #define RESTORE_COMMAND_COPY 0
  68 #define RESTORE_COMMAND_LINK 1
  69 int                     restoreCommandType;
  70
  71 #define XLOG_DATA                        0
  72 #define XLOG_HISTORY             1
  73 #define XLOG_BACKUP_LABEL        2
  74 int                     nextWALFileType;
  75
  76 #define SET_RESTORE_COMMAND(cmd, arg1, arg2) \
  77         snprintf(restoreCommand, MAXPGPATH, cmd " \"%s\" \"%s\"", arg1, arg2)
  78
  79 struct stat stat_buf;
  80
  81 /* =====================================================================
  82  *
  83  *                Customizable section
  84  *
  85  * =====================================================================
  86  *
  87  *      Currently, this section assumes that the Archive is a locally
  88  *      accessible directory. If you want to make other assumptions,
  89  *      such as using a vendor-specific archive and access API, these
  90  *      routines are the ones you'll need to change. You're
  91  *      enouraged to submit any changes to pgsql-patches@postgresql.org
  92  *      or personally to the current maintainer. Those changes may be
  93  *      folded in to later versions of this program.
  94  */
  95
  96 #define XLOG_DATA_FNAME_LEN             24
  97 /* Reworked from access/xlog_internal.h */
  98 #define XLogFileName(fname, tli, log, seg)      \
  99         snprintf(fname, XLOG_DATA_FNAME_LEN + 1, "%08X%08X%08X", tli, log, seg)
 100
 101 /*
 102  *      Initialize allows customized commands into the warm standby program.
 103  *
 104  *      As an example, and probably the common case, we use either
 105  *      cp/ln commands on *nix, or copy/move command on Windows.
 106  *
 107  */
 108 static void
 109 CustomizableInitialize(void)
 110 {
 111 #ifdef WIN32
 112         snprintf(WALFilePath, MAXPGPATH, "%s\\%s", archiveLocation, nextWALFileName);
 113         switch (restoreCommandType)
 114         {
 115                 case RESTORE_COMMAND_LINK:
 116                         SET_RESTORE_COMMAND("mklink", WALFilePath, xlogFilePath);
 117                 case RESTORE_COMMAND_COPY:
 118                 default:
 119                         SET_RESTORE_COMMAND("copy", WALFilePath, xlogFilePath);
 120                         break;
 121         }
 122 #else
 123         snprintf(WALFilePath, MAXPGPATH, "%s/%s", archiveLocation, nextWALFileName);
 124         switch (restoreCommandType)
 125         {
 126                 case RESTORE_COMMAND_LINK:
 127 #if HAVE_WORKING_LINK
 128                         SET_RESTORE_COMMAND("ln -s -f", WALFilePath, xlogFilePath);
 129                         break;
 130 #endif
 131                 case RESTORE_COMMAND_COPY:
 132                 default:
 133                         SET_RESTORE_COMMAND("cp", WALFilePath, xlogFilePath);
 134                         break;
 135         }
 136 #endif
 137
 138         /*
 139          * This code assumes that archiveLocation is a directory You may wish to
 140          * add code to check for tape libraries, etc.. So, since it is a
 141          * directory, we use stat to test if its accessible
 142          */
 143         if (stat(archiveLocation, &stat_buf) != 0)
 144         {
 145                 fprintf(stderr, "pg_standby: archiveLocation \"%s\" does not exist\n", archiveLocation);
 146                 fflush(stderr);
 147                 exit(2);
 148         }
 149 }
 150
 151 /*
 152  * CustomizableNextWALFileReady()
 153  *
 154  *        Is the requested file ready yet?
 155  */
 156 static bool
 157 CustomizableNextWALFileReady()
 158 {
 159         if (stat(WALFilePath, &stat_buf) == 0)
 160         {
 161                 /*
 162                  * If its a backup file, return immediately If its a regular file
 163                  * return only if its the right size already
 164                  */
 165                 if (strlen(nextWALFileName) > 24 &&
 166                         strspn(nextWALFileName, "0123456789ABCDEF") == 24 &&
 167                 strcmp(nextWALFileName + strlen(nextWALFileName) - strlen(".backup"),
 168                            ".backup") == 0)
 169                 {
 170                         nextWALFileType = XLOG_BACKUP_LABEL;
 171                         return true;
 172                 }
 173                 else if (stat_buf.st_size == XLOG_SEG_SIZE)
 174                 {
 175 #ifdef WIN32
 176
 177                         /*
 178                          * Windows reports that the file has the right number of bytes
 179                          * even though the file is still being copied and cannot be opened
 180                          * by pg_standby yet. So we wait for sleeptime secs before
 181                          * attempting to restore. If that is not enough, we will rely on
 182                          * the retry/holdoff mechanism.
 183                          */
 184                         pg_usleep(sleeptime * 1000000L);
 185 #endif
 186                         nextWALFileType = XLOG_DATA;
 187                         return true;
 188                 }
 189
 190                 /*
 191                  * If still too small, wait until it is the correct size
 192                  */
 193                 if (stat_buf.st_size > XLOG_SEG_SIZE)
 194                 {
 195                         if (debug)
 196                         {
 197                                 fprintf(stderr, "file size greater than expected\n");
 198                                 fflush(stderr);
 199                         }
 200                         exit(3);
 201                 }
 202         }
 203
 204         return false;
 205 }
 206
 207 #define MaxSegmentsPerLogFile ( 0xFFFFFFFF / XLOG_SEG_SIZE )
 208
 209 static void
 210 CustomizableCleanupPriorWALFiles(void)
 211 {
 212         /*
 213          * Work out name of prior file from current filename
 214          */
 215         if (nextWALFileType == XLOG_DATA)
 216         {
 217                 int                     rc;
 218                 DIR                *xldir;
 219                 struct dirent *xlde;
 220
 221                 /*
 222                  * Assume its OK to keep failing. The failure situation may change
 223                  * over time, so we'd rather keep going on the main processing than
 224                  * fail because we couldnt clean up yet.
 225                  */
 226                 if ((xldir = opendir(archiveLocation)) != NULL)
 227                 {
 228                         while ((xlde = readdir(xldir)) != NULL)
 229                         {
 230                                 /*
 231                                  * We ignore the timeline part of the XLOG segment identifiers
 232                                  * in deciding whether a segment is still needed.  This
 233                                  * ensures that we won't prematurely remove a segment from a
 234                                  * parent timeline. We could probably be a little more
 235                                  * proactive about removing segments of non-parent timelines,
 236                                  * but that would be a whole lot more complicated.
 237                                  *
 238                                  * We use the alphanumeric sorting property of the filenames
 239                                  * to decide which ones are earlier than the
 240                                  * exclusiveCleanupFileName file. Note that this means files
 241                                  * are not removed in the order they were originally written,
 242                                  * in case this worries you.
 243                                  */
 244                                 if (strlen(xlde->d_name) == XLOG_DATA_FNAME_LEN &&
 245                                         strspn(xlde->d_name, "0123456789ABCDEF") == XLOG_DATA_FNAME_LEN &&
 246                                   strcmp(xlde->d_name + 8, exclusiveCleanupFileName + 8) < 0)
 247                                 {
 248 #ifdef WIN32
 249                                         snprintf(WALFilePath, MAXPGPATH, "%s\\%s", archiveLocation, xlde->d_name);
 250 #else
 251                                         snprintf(WALFilePath, MAXPGPATH, "%s/%s", archiveLocation, xlde->d_name);
 252 #endif
 253
 254                                         if (debug)
 255                                                 fprintf(stderr, "\nremoving \"%s\"", WALFilePath);
 256
 257                                         rc = unlink(WALFilePath);
 258                                         if (rc != 0)
 259                                         {
 260                                                 fprintf(stderr, "\npg_standby: ERROR failed to remove \"%s\": %s",
 261                                                                 WALFilePath, strerror(errno));
 262                                                 break;
 263                                         }
 264                                 }
 265                         }
 266                         if (debug)
 267                                 fprintf(stderr, "\n");
 268                 }
 269                 else
 270                         fprintf(stderr, "pg_standby: archiveLocation \"%s\" open error\n", archiveLocation);
 271
 272                 closedir(xldir);
 273                 fflush(stderr);
 274         }
 275 }
 276
 277 /* =====================================================================
 278  *                End of Customizable section
 279  * =====================================================================
 280  */
 281
 282 /*
 283  * SetWALFileNameForCleanup()
 284  *
 285  *        Set the earliest WAL filename that we want to keep on the archive
 286  *        and decide whether we need_cleanup
 287  */
 288 static bool
 289 SetWALFileNameForCleanup(void)
 290 {
 291         uint32          tli = 1,
 292                                 log = 0,
 293                                 seg = 0;
 294         uint32          log_diff = 0,
 295                                 seg_diff = 0;
 296         bool            cleanup = false;
 297
 298         if (restartWALFileName)
 299         {
 300                 strcpy(exclusiveCleanupFileName, restartWALFileName);
 301                 return true;
 302         }
 303
 304         if (keepfiles > 0)
 305         {
 306                 sscanf(nextWALFileName, "%08X%08X%08X", &tli, &log, &seg);
 307                 if (tli > 0 && log >= 0 && seg > 0)
 308                 {
 309                         log_diff = keepfiles / MaxSegmentsPerLogFile;
 310                         seg_diff = keepfiles % MaxSegmentsPerLogFile;
 311                         if (seg_diff > seg)
 312                         {
 313                                 log_diff++;
 314                                 seg = MaxSegmentsPerLogFile - seg_diff;
 315                         }
 316                         else
 317                                 seg -= seg_diff;
 318
 319                         if (log >= log_diff)
 320                         {
 321                                 log -= log_diff;
 322                                 cleanup = true;
 323                         }
 324                         else
 325                         {
 326                                 log = 0;
 327                                 seg = 0;
 328                         }
 329                 }
 330         }
 331
 332         XLogFileName(exclusiveCleanupFileName, tli, log, seg);
 333
 334         return cleanup;
 335 }
 336
 337 /*
 338  * CheckForExternalTrigger()
 339  *
 340  *        Is there a trigger file?
 341  */
 342 static bool
 343 CheckForExternalTrigger(void)
 344 {
 345         int                     rc;
 346
 347         /*
 348          * Look for a trigger file, if that option has been selected
 349          *
 350          * We use stat() here because triggerPath is always a file rather than
 351          * potentially being in an archive
 352          */
 353         if (triggerPath && stat(triggerPath, &stat_buf) == 0)
 354         {
 355                 fprintf(stderr, "trigger file found\n");
 356                 fflush(stderr);
 357
 358                 /*
 359                  * If trigger file found, we *must* delete it. Here's why: When
 360                  * recovery completes, we will be asked again for the same file from
 361                  * the archive using pg_standby so must remove trigger file so we can
 362                  * reload file again and come up correctly.
 363                  */
 364                 rc = unlink(triggerPath);
 365                 if (rc != 0)
 366                 {
 367                         fprintf(stderr, "\n ERROR: could not remove \"%s\": %s", triggerPath, strerror(errno));
 368                         fflush(stderr);
 369                         exit(rc);
 370                 }
 371                 return true;
 372         }
 373
 374         return false;
 375 }
 376
 377 /*
 378  * RestoreWALFileForRecovery()
 379  *
 380  *        Perform the action required to restore the file from archive
 381  */
 382 static bool
 383 RestoreWALFileForRecovery(void)
 384 {
 385         int                     rc = 0;
 386         int                     numretries = 0;
 387
 388         if (debug)
 389         {
 390                 fprintf(stderr, "\nrunning restore              :");
 391                 fflush(stderr);
 392         }
 393
 394         while (numretries < maxretries)
 395         {
 396                 rc = system(restoreCommand);
 397                 if (rc == 0)
 398                 {
 399                         if (debug)
 400                         {
 401                                 fprintf(stderr, " OK");
 402                                 fflush(stderr);
 403                         }
 404                         return true;
 405                 }
 406                 pg_usleep(numretries++ * sleeptime * 1000000L);
 407         }
 408
 409         /*
 410          * Allow caller to add additional info
 411          */
 412         if (debug)
 413                 fprintf(stderr, "not restored           : ");
 414         return false;
 415 }
 416
 417 static void
 418 usage(void)
 419 {
 420         fprintf(stderr, "\npg_standby allows Warm Standby servers to be configured\n");
 421         fprintf(stderr, "Usage:\n");
 422         fprintf(stderr, "  pg_standby [OPTION]... ARCHIVELOCATION NEXTWALFILE XLOGFILEPATH [RESTARTWALFILE]\n");
 423         fprintf(stderr, "                               note space between ARCHIVELOCATION and NEXTWALFILE\n");
 424         fprintf(stderr, "with main intended use as a restore_command in the recovery.conf\n");
 425         fprintf(stderr, "        restore_command = 'pg_standby [OPTION]... ARCHIVELOCATION %%f %%p %%r'\n");
 426         fprintf(stderr, "e.g. restore_command = 'pg_standby -l /mnt/server/archiverdir %%f %%p %%r'\n");
 427         fprintf(stderr, "\nOptions:\n");
 428         fprintf(stderr, "  -c                   copies file from archive (default)\n");
 429         fprintf(stderr, "  -d                   generate lots of debugging output (testing only)\n");
 430         fprintf(stderr, "  -k NUMFILESTOKEEP    if RESTARTWALFILE not used, removes files prior to limit (0 keeps all)\n");
 431         fprintf(stderr, "  -l                   links into archive (leaves file in archive)\n");
 432         fprintf(stderr, "  -r MAXRETRIES                max number of times to retry, with progressive wait (default=3)\n");
 433         fprintf(stderr, "  -s SLEEPTIME         seconds to wait between file checks (min=1, max=60, default=5)\n");
 434         fprintf(stderr, "  -t TRIGGERFILE       defines a trigger file to initiate failover (no default)\n");
 435         fprintf(stderr, "  -w MAXWAITTIME       max seconds to wait for a file (0=no limit)(default=0)\n");
 436         fflush(stderr);
 437 }
 438
 439 static void
 440 sighandler(int sig)
 441 {
 442         signaled = true;
 443 }
 444
 445 /*------------ MAIN ----------------------------------------*/
 446 int
 447 main(int argc, char **argv)
 448 {
 449         int                     c;
 450
 451         (void) signal(SIGINT, sighandler);
 452         (void) signal(SIGQUIT, sighandler);
 453
 454         while ((c = getopt(argc, argv, "cdk:lr:s:t:w:")) != -1)
 455         {
 456                 switch (c)
 457                 {
 458                         case 'c':                       /* Use copy */
 459                                 restoreCommandType = RESTORE_COMMAND_COPY;
 460                                 break;
 461                         case 'd':                       /* Debug mode */
 462                                 debug = true;
 463                                 break;
 464                         case 'k':                       /* keepfiles */
 465                                 keepfiles = atoi(optarg);
 466                                 if (keepfiles < 0)
 467                                 {
 468                                         fprintf(stderr, "usage: pg_standby -k keepfiles must be >= 0\n");
 469                                         usage();
 470                                         exit(2);
 471                                 }
 472                                 break;
 473                         case 'l':                       /* Use link */
 474                                 restoreCommandType = RESTORE_COMMAND_LINK;
 475                                 break;
 476                         case 'r':                       /* Retries */
 477                                 maxretries = atoi(optarg);
 478                                 if (maxretries < 0)
 479                                 {
 480                                         fprintf(stderr, "usage: pg_standby -r maxretries must be >= 0\n");
 481                                         usage();
 482                                         exit(2);
 483                                 }
 484                                 break;
 485                         case 's':                       /* Sleep time */
 486                                 sleeptime = atoi(optarg);
 487                                 if (sleeptime <= 0 || sleeptime > 60)
 488                                 {
 489                                         fprintf(stderr, "usage: pg_standby -s sleeptime incorrectly set\n");
 490                                         usage();
 491                                         exit(2);
 492                                 }
 493                                 break;
 494                         case 't':                       /* Trigger file */
 495                                 triggerPath = optarg;
 496                                 if (CheckForExternalTrigger())
 497                                         exit(1);        /* Normal exit, with non-zero */
 498                                 break;
 499                         case 'w':                       /* Max wait time */
 500                                 maxwaittime = atoi(optarg);
 501                                 if (maxwaittime < 0)
 502                                 {
 503                                         fprintf(stderr, "usage: pg_standby -w maxwaittime incorrectly set\n");
 504                                         usage();
 505                                         exit(2);
 506                                 }
 507                                 break;
 508                         default:
 509                                 usage();
 510                                 exit(2);
 511                                 break;
 512                 }
 513         }
 514
 515         /*
 516          * Parameter checking - after checking to see if trigger file present
 517          */
 518         if (argc == 1)
 519         {
 520                 usage();
 521                 exit(2);
 522         }
 523
 524         /*
 525          * We will go to the archiveLocation to get nextWALFileName.
 526          * nextWALFileName may not exist yet, which would not be an error, so we
 527          * separate the archiveLocation and nextWALFileName so we can check
 528          * separately whether archiveLocation exists, if not that is an error
 529          */
 530         if (optind < argc)
 531         {
 532                 archiveLocation = argv[optind];
 533                 optind++;
 534         }
 535         else
 536         {
 537                 fprintf(stderr, "pg_standby: must specify archiveLocation\n");
 538                 usage();
 539                 exit(2);
 540         }
 541
 542         if (optind < argc)
 543         {
 544                 nextWALFileName = argv[optind];
 545                 optind++;
 546         }
 547         else
 548         {
 549                 fprintf(stderr, "pg_standby: use %%f to specify nextWALFileName\n");
 550                 usage();
 551                 exit(2);
 552         }
 553
 554         if (optind < argc)
 555         {
 556                 xlogFilePath = argv[optind];
 557                 optind++;
 558         }
 559         else
 560         {
 561                 fprintf(stderr, "pg_standby: use %%p to specify xlogFilePath\n");
 562                 usage();
 563                 exit(2);
 564         }
 565
 566         if (optind < argc)
 567         {
 568                 restartWALFileName = argv[optind];
 569                 optind++;
 570         }
 571
 572         CustomizableInitialize();
 573
 574         need_cleanup = SetWALFileNameForCleanup();
 575
 576         if (debug)
 577         {
 578                 fprintf(stderr, "\nTrigger file                 : %s", triggerPath ? triggerPath : "<not set>");
 579                 fprintf(stderr, "\nWaiting for WAL file : %s", nextWALFileName);
 580                 fprintf(stderr, "\nWAL file path                : %s", WALFilePath);
 581                 fprintf(stderr, "\nRestoring to...              : %s", xlogFilePath);
 582                 fprintf(stderr, "\nSleep interval               : %d second%s",
 583                                 sleeptime, (sleeptime > 1 ? "s" : " "));
 584                 fprintf(stderr, "\nMax wait interval    : %d %s",
 585                                 maxwaittime, (maxwaittime > 0 ? "seconds" : "forever"));
 586                 fprintf(stderr, "\nCommand for restore  : %s", restoreCommand);
 587                 fprintf(stderr, "\nKeep archive history : %s and later", exclusiveCleanupFileName);
 588                 fflush(stderr);
 589         }
 590
 591         /*
 592          * Check for initial history file: always the first file to be requested
 593          * It's OK if the file isn't there - all other files need to wait
 594          */
 595         if (strlen(nextWALFileName) > 8 &&
 596                 strspn(nextWALFileName, "0123456789ABCDEF") == 8 &&
 597                 strcmp(nextWALFileName + strlen(nextWALFileName) - strlen(".history"),
 598                            ".history") == 0)
 599         {
 600                 nextWALFileType = XLOG_HISTORY;
 601                 if (RestoreWALFileForRecovery())
 602                         exit(0);
 603                 else
 604                 {
 605                         if (debug)
 606                         {
 607                                 fprintf(stderr, "history file not found\n");
 608                                 fflush(stderr);
 609                         }
 610                         exit(1);
 611                 }
 612         }
 613
 614         /*
 615          * Main wait loop
 616          */
 617         while (!CustomizableNextWALFileReady() && !triggered)
 618         {
 619                 if (sleeptime <= 60)
 620                         pg_usleep(sleeptime * 1000000L);
 621
 622                 if (signaled)
 623                 {
 624                         triggered = true;
 625                         if (debug)
 626                         {
 627                                 fprintf(stderr, "\nsignaled to exit\n");
 628                                 fflush(stderr);
 629                         }
 630                 }
 631                 else
 632                 {
 633
 634                         if (debug)
 635                         {
 636                                 fprintf(stderr, "\nWAL file not present yet.");
 637                                 if (triggerPath)
 638                                         fprintf(stderr, " Checking for trigger file...");
 639                                 fflush(stderr);
 640                         }
 641
 642                         waittime += sleeptime;
 643
 644                         if (!triggered && (CheckForExternalTrigger() || (waittime >= maxwaittime && maxwaittime > 0)))
 645                         {
 646                                 triggered = true;
 647                                 if (debug && waittime >= maxwaittime && maxwaittime > 0)
 648                                         fprintf(stderr, "\nTimed out after %d seconds\n", waittime);
 649                         }
 650                 }
 651         }
 652
 653         /*
 654          * Action on exit
 655          */
 656         if (triggered)
 657                 exit(1);                                /* Normal exit, with non-zero */
 658
 659         /*
 660          * Once we have restored this file successfully we can remove some prior
 661          * WAL files. If this restore fails we musn't remove any file because some
 662          * of them will be requested again immediately after the failed restore,
 663          * or when we restart recovery.
 664          */
 665         if (RestoreWALFileForRecovery() && need_cleanup)
 666                 CustomizableCleanupPriorWALFiles();
 667
 668         return 0;
 669 }