contrib/pg_standby/pg_standby.c

   1 /*
   2  * $PostgreSQL$
   3  *
   4  *
   5  * pg_standby.c
   6  *
   7  * Production-ready example of how to create a Warm Standby
   8  * database server using continuous archiving as a
   9  * replication mechanism
  10  *
  11  * We separate the parameters for archive and nextWALfile
  12  * so that we can check the archive exists, even if the
  13  * WAL file doesn't (yet).
  14  *
  15  * This program will be executed once in full for each file
  16  * requested by the warm standby server.
  17  *
  18  * It is designed to cater to a variety of needs, as well
  19  * providing a customizable section.
  20  *
  21  * Original author:             Simon Riggs  simon@2ndquadrant.com
  22  * Current maintainer:  Simon Riggs
  23  */
  24 #include "postgres_fe.h"
  25
  26 #include <ctype.h>
  27 #include <dirent.h>
  28 #include <sys/stat.h>
  29 #include <signal.h>
  30
  31 #ifdef WIN32
  32 int                     getopt(int argc, char *const argv[], const char *optstring);
  33 #else
  34 #include <sys/time.h>
  35 #include <unistd.h>
  36
  37 #ifdef HAVE_GETOPT_H
  38 #include <getopt.h>
  39 #endif
  40 #endif   /* ! WIN32 */
  41
  42 extern char *optarg;
  43 extern int      optind;
  44
  45 /* Options and defaults */
  46 int                     sleeptime = 5;          /* amount of time to sleep between file checks */
  47 int                     waittime = -1;          /* how long we have been waiting, -1 no wait
  48                                                                  * yet */
  49 int                     maxwaittime = 0;        /* how long are we prepared to wait for? */
  50 int                     keepfiles = 0;          /* number of WAL files to keep, 0 keep all */
  51 int                     maxretries = 3;         /* number of retries on restore command */
  52 bool            debug = false;          /* are we debugging? */
  53 bool            triggered = false;      /* have we been triggered? */
  54 bool            need_cleanup = false;           /* do we need to remove files from
  55                                                                                  * archive? */
  56
  57 static volatile sig_atomic_t signaled = false;
  58
  59 char       *archiveLocation;    /* where to find the archive? */
  60 char       *triggerPath;                /* where to find the trigger file? */
  61 char       *xlogFilePath;               /* where we are going to restore to */
  62 char       *nextWALFileName;    /* the file we need to get from archive */
  63 char       *restartWALFileName; /* the file from which we can restart restore */
  64 char       *priorWALFileName;   /* the file we need to get from archive */
  65 char            WALFilePath[MAXPGPATH];         /* the file path including archive */
  66 char            restoreCommand[MAXPGPATH];      /* run this to restore */
  67 char            exclusiveCleanupFileName[MAXPGPATH];            /* the file we need to
  68                                                                                                                  * get from archive */
  69
  70 #define RESTORE_COMMAND_COPY 0
  71 #define RESTORE_COMMAND_LINK 1
  72 int                     restoreCommandType;
  73
  74 #define XLOG_DATA                        0
  75 #define XLOG_HISTORY             1
  76 #define XLOG_BACKUP_LABEL        2
  77 int                     nextWALFileType;
  78
  79 #define SET_RESTORE_COMMAND(cmd, arg1, arg2) \
  80         snprintf(restoreCommand, MAXPGPATH, cmd " \"%s\" \"%s\"", arg1, arg2)
  81
  82 struct stat stat_buf;
  83
  84 /* =====================================================================
  85  *
  86  *                Customizable section
  87  *
  88  * =====================================================================
  89  *
  90  *      Currently, this section assumes that the Archive is a locally
  91  *      accessible directory. If you want to make other assumptions,
  92  *      such as using a vendor-specific archive and access API, these
  93  *      routines are the ones you'll need to change. You're
  94  *      enouraged to submit any changes to pgsql-patches@postgresql.org
  95  *      or personally to the current maintainer. Those changes may be
  96  *      folded in to later versions of this program.
  97  */
  98
  99 #define XLOG_DATA_FNAME_LEN             24
 100 /* Reworked from access/xlog_internal.h */
 101 #define XLogFileName(fname, tli, log, seg)      \
 102         snprintf(fname, XLOG_DATA_FNAME_LEN + 1, "%08X%08X%08X", tli, log, seg)
 103
 104 /*
 105  *      Initialize allows customized commands into the warm standby program.
 106  *
 107  *      As an example, and probably the common case, we use either
 108  *      cp/ln commands on *nix, or copy/move command on Windows.
 109  *
 110  */
 111 static void
 112 CustomizableInitialize(void)
 113 {
 114 #ifdef WIN32
 115         snprintf(WALFilePath, MAXPGPATH, "%s\\%s", archiveLocation, nextWALFileName);
 116         switch (restoreCommandType)
 117         {
 118                 case RESTORE_COMMAND_LINK:
 119                         SET_RESTORE_COMMAND("mklink", WALFilePath, xlogFilePath);
 120                         break;
 121                 case RESTORE_COMMAND_COPY:
 122                 default:
 123                         SET_RESTORE_COMMAND("copy", WALFilePath, xlogFilePath);
 124                         break;
 125         }
 126 #else
 127         snprintf(WALFilePath, MAXPGPATH, "%s/%s", archiveLocation, nextWALFileName);
 128         switch (restoreCommandType)
 129         {
 130                 case RESTORE_COMMAND_LINK:
 131 #if HAVE_WORKING_LINK
 132                         SET_RESTORE_COMMAND("ln -s -f", WALFilePath, xlogFilePath);
 133                         break;
 134 #endif
 135                 case RESTORE_COMMAND_COPY:
 136                 default:
 137                         SET_RESTORE_COMMAND("cp", WALFilePath, xlogFilePath);
 138                         break;
 139         }
 140 #endif
 141
 142         /*
 143          * This code assumes that archiveLocation is a directory You may wish to
 144          * add code to check for tape libraries, etc.. So, since it is a
 145          * directory, we use stat to test if its accessible
 146          */
 147         if (stat(archiveLocation, &stat_buf) != 0)
 148         {
 149                 fprintf(stderr, "pg_standby: archiveLocation \"%s\" does not exist\n", archiveLocation);
 150                 fflush(stderr);
 151                 exit(2);
 152         }
 153 }
 154
 155 /*
 156  * CustomizableNextWALFileReady()
 157  *
 158  *        Is the requested file ready yet?
 159  */
 160 static bool
 161 CustomizableNextWALFileReady()
 162 {
 163         if (stat(WALFilePath, &stat_buf) == 0)
 164         {
 165                 /*
 166                  * If its a backup file, return immediately If its a regular file
 167                  * return only if its the right size already
 168                  */
 169                 if (strlen(nextWALFileName) > 24 &&
 170                         strspn(nextWALFileName, "0123456789ABCDEF") == 24 &&
 171                 strcmp(nextWALFileName + strlen(nextWALFileName) - strlen(".backup"),
 172                            ".backup") == 0)
 173                 {
 174                         nextWALFileType = XLOG_BACKUP_LABEL;
 175                         return true;
 176                 }
 177                 else if (stat_buf.st_size == XLOG_SEG_SIZE)
 178                 {
 179 #ifdef WIN32
 180
 181                         /*
 182                          * Windows reports that the file has the right number of bytes
 183                          * even though the file is still being copied and cannot be opened
 184                          * by pg_standby yet. So we wait for sleeptime secs before
 185                          * attempting to restore. If that is not enough, we will rely on
 186                          * the retry/holdoff mechanism.  GNUWin32's cp does not have
 187                          * this problem.
 188                          */
 189                         pg_usleep(sleeptime * 1000000L);
 190 #endif
 191                         nextWALFileType = XLOG_DATA;
 192                         return true;
 193                 }
 194
 195                 /*
 196                  * If still too small, wait until it is the correct size
 197                  */
 198                 if (stat_buf.st_size > XLOG_SEG_SIZE)
 199                 {
 200                         if (debug)
 201                         {
 202                                 fprintf(stderr, "file size greater than expected\n");
 203                                 fflush(stderr);
 204                         }
 205                         exit(3);
 206                 }
 207         }
 208
 209         return false;
 210 }
 211
 212 #define MaxSegmentsPerLogFile ( 0xFFFFFFFF / XLOG_SEG_SIZE )
 213
 214 static void
 215 CustomizableCleanupPriorWALFiles(void)
 216 {
 217         /*
 218          * Work out name of prior file from current filename
 219          */
 220         if (nextWALFileType == XLOG_DATA)
 221         {
 222                 int                     rc;
 223                 DIR                *xldir;
 224                 struct dirent *xlde;
 225
 226                 /*
 227                  * Assume its OK to keep failing. The failure situation may change
 228                  * over time, so we'd rather keep going on the main processing than
 229                  * fail because we couldnt clean up yet.
 230                  */
 231                 if ((xldir = opendir(archiveLocation)) != NULL)
 232                 {
 233                         while ((xlde = readdir(xldir)) != NULL)
 234                         {
 235                                 /*
 236                                  * We ignore the timeline part of the XLOG segment identifiers
 237                                  * in deciding whether a segment is still needed.  This
 238                                  * ensures that we won't prematurely remove a segment from a
 239                                  * parent timeline. We could probably be a little more
 240                                  * proactive about removing segments of non-parent timelines,
 241                                  * but that would be a whole lot more complicated.
 242                                  *
 243                                  * We use the alphanumeric sorting property of the filenames
 244                                  * to decide which ones are earlier than the
 245                                  * exclusiveCleanupFileName file. Note that this means files
 246                                  * are not removed in the order they were originally written,
 247                                  * in case this worries you.
 248                                  */
 249                                 if (strlen(xlde->d_name) == XLOG_DATA_FNAME_LEN &&
 250                                         strspn(xlde->d_name, "0123456789ABCDEF") == XLOG_DATA_FNAME_LEN &&
 251                                   strcmp(xlde->d_name + 8, exclusiveCleanupFileName + 8) < 0)
 252                                 {
 253 #ifdef WIN32
 254                                         snprintf(WALFilePath, MAXPGPATH, "%s\\%s", archiveLocation, xlde->d_name);
 255 #else
 256                                         snprintf(WALFilePath, MAXPGPATH, "%s/%s", archiveLocation, xlde->d_name);
 257 #endif
 258
 259                                         if (debug)
 260                                                 fprintf(stderr, "\nremoving \"%s\"", WALFilePath);
 261
 262                                         rc = unlink(WALFilePath);
 263                                         if (rc != 0)
 264                                         {
 265                                                 fprintf(stderr, "\npg_standby: ERROR failed to remove \"%s\": %s",
 266                                                                 WALFilePath, strerror(errno));
 267                                                 break;
 268                                         }
 269                                 }
 270                         }
 271                         if (debug)
 272                                 fprintf(stderr, "\n");
 273                 }
 274                 else
 275                         fprintf(stderr, "pg_standby: archiveLocation \"%s\" open error\n", archiveLocation);
 276
 277                 closedir(xldir);
 278                 fflush(stderr);
 279         }
 280 }
 281
 282 /* =====================================================================
 283  *                End of Customizable section
 284  * =====================================================================
 285  */
 286
 287 /*
 288  * SetWALFileNameForCleanup()
 289  *
 290  *        Set the earliest WAL filename that we want to keep on the archive
 291  *        and decide whether we need_cleanup
 292  */
 293 static bool
 294 SetWALFileNameForCleanup(void)
 295 {
 296         uint32          tli = 1,
 297                                 log = 0,
 298                                 seg = 0;
 299         uint32          log_diff = 0,
 300                                 seg_diff = 0;
 301         bool            cleanup = false;
 302
 303         if (restartWALFileName)
 304         {
 305                 /*
 306                  * Don't do cleanup if the restartWALFileName provided
 307                  * is later than the xlog file requested. This is an error
 308                  * and we must not remove these files from archive.
 309                  * This shouldn't happen, but better safe than sorry.
 310                  */
 311                 if (strcmp(restartWALFileName, nextWALFileName) > 0)
 312                         return false;
 313
 314                 strcpy(exclusiveCleanupFileName, restartWALFileName);
 315                 return true;
 316         }
 317
 318         if (keepfiles > 0)
 319         {
 320                 sscanf(nextWALFileName, "%08X%08X%08X", &tli, &log, &seg);
 321                 if (tli > 0 && log >= 0 && seg > 0)
 322                 {
 323                         log_diff = keepfiles / MaxSegmentsPerLogFile;
 324                         seg_diff = keepfiles % MaxSegmentsPerLogFile;
 325                         if (seg_diff > seg)
 326                         {
 327                                 log_diff++;
 328                                 seg = MaxSegmentsPerLogFile - (seg_diff - seg);
 329                         }
 330                         else
 331                                 seg -= seg_diff;
 332
 333                         if (log >= log_diff)
 334                         {
 335                                 log -= log_diff;
 336                                 cleanup = true;
 337                         }
 338                         else
 339                         {
 340                                 log = 0;
 341                                 seg = 0;
 342                         }
 343                 }
 344         }
 345
 346         XLogFileName(exclusiveCleanupFileName, tli, log, seg);
 347
 348         return cleanup;
 349 }
 350
 351 /*
 352  * CheckForExternalTrigger()
 353  *
 354  *        Is there a trigger file?
 355  */
 356 static bool
 357 CheckForExternalTrigger(void)
 358 {
 359         int                     rc;
 360
 361         /*
 362          * Look for a trigger file, if that option has been selected
 363          *
 364          * We use stat() here because triggerPath is always a file rather than
 365          * potentially being in an archive
 366          */
 367         if (triggerPath && stat(triggerPath, &stat_buf) == 0)
 368         {
 369                 fprintf(stderr, "trigger file found\n");
 370                 fflush(stderr);
 371
 372                 /*
 373                  * If trigger file found, we *must* delete it. Here's why: When
 374                  * recovery completes, we will be asked again for the same file from
 375                  * the archive using pg_standby so must remove trigger file so we can
 376                  * reload file again and come up correctly.
 377                  */
 378                 rc = unlink(triggerPath);
 379                 if (rc != 0)
 380                 {
 381                         fprintf(stderr, "\n ERROR: could not remove \"%s\": %s", triggerPath, strerror(errno));
 382                         fflush(stderr);
 383                         exit(rc);
 384                 }
 385                 return true;
 386         }
 387
 388         return false;
 389 }
 390
 391 /*
 392  * RestoreWALFileForRecovery()
 393  *
 394  *        Perform the action required to restore the file from archive
 395  */
 396 static bool
 397 RestoreWALFileForRecovery(void)
 398 {
 399         int                     rc = 0;
 400         int                     numretries = 0;
 401
 402         if (debug)
 403         {
 404                 fprintf(stderr, "\nrunning restore              :");
 405                 fflush(stderr);
 406         }
 407
 408         while (numretries < maxretries)
 409         {
 410                 rc = system(restoreCommand);
 411                 if (rc == 0)
 412                 {
 413                         if (debug)
 414                         {
 415                                 fprintf(stderr, " OK");
 416                                 fflush(stderr);
 417                         }
 418                         return true;
 419                 }
 420                 pg_usleep(numretries++ * sleeptime * 1000000L);
 421         }
 422
 423         /*
 424          * Allow caller to add additional info
 425          */
 426         if (debug)
 427                 fprintf(stderr, "not restored           : ");
 428         return false;
 429 }
 430
 431 static void
 432 usage(void)
 433 {
 434         fprintf(stderr, "\npg_standby allows Warm Standby servers to be configured\n");
 435         fprintf(stderr, "Usage:\n");
 436         fprintf(stderr, "  pg_standby [OPTION]... ARCHIVELOCATION NEXTWALFILE XLOGFILEPATH [RESTARTWALFILE]\n");
 437         fprintf(stderr, "                               note space between ARCHIVELOCATION and NEXTWALFILE\n");
 438         fprintf(stderr, "with main intended use as a restore_command in the recovery.conf\n");
 439         fprintf(stderr, "        restore_command = 'pg_standby [OPTION]... ARCHIVELOCATION %%f %%p %%r'\n");
 440         fprintf(stderr, "e.g. restore_command = 'pg_standby -l /mnt/server/archiverdir %%f %%p %%r'\n");
 441         fprintf(stderr, "\nOptions:\n");
 442         fprintf(stderr, "  -c                   copies file from archive (default)\n");
 443         fprintf(stderr, "  -d                   generate lots of debugging output (testing only)\n");
 444         fprintf(stderr, "  -k NUMFILESTOKEEP    if RESTARTWALFILE not used, removes files prior to limit (0 keeps all)\n");
 445         fprintf(stderr, "  -l                   links into archive (leaves file in archive)\n");
 446         fprintf(stderr, "  -r MAXRETRIES                max number of times to retry, with progressive wait (default=3)\n");
 447         fprintf(stderr, "  -s SLEEPTIME         seconds to wait between file checks (min=1, max=60, default=5)\n");
 448         fprintf(stderr, "  -t TRIGGERFILE       defines a trigger file to initiate failover (no default)\n");
 449         fprintf(stderr, "  -w MAXWAITTIME       max seconds to wait for a file (0=no limit)(default=0)\n");
 450         fflush(stderr);
 451 }
 452
 453 static void
 454 sighandler(int sig)
 455 {
 456         signaled = true;
 457 }
 458
 459 /*------------ MAIN ----------------------------------------*/
 460 int
 461 main(int argc, char **argv)
 462 {
 463         int                     c;
 464
 465         (void) signal(SIGINT, sighandler);
 466         (void) signal(SIGQUIT, sighandler);
 467
 468         while ((c = getopt(argc, argv, "cdk:lr:s:t:w:")) != -1)
 469         {
 470                 switch (c)
 471                 {
 472                         case 'c':                       /* Use copy */
 473                                 restoreCommandType = RESTORE_COMMAND_COPY;
 474                                 break;
 475                         case 'd':                       /* Debug mode */
 476                                 debug = true;
 477                                 break;
 478                         case 'k':                       /* keepfiles */
 479                                 keepfiles = atoi(optarg);
 480                                 if (keepfiles < 0)
 481                                 {
 482                                         fprintf(stderr, "usage: pg_standby -k keepfiles must be >= 0\n");
 483                                         usage();
 484                                         exit(2);
 485                                 }
 486                                 break;
 487                         case 'l':                       /* Use link */
 488                                 restoreCommandType = RESTORE_COMMAND_LINK;
 489                                 break;
 490                         case 'r':                       /* Retries */
 491                                 maxretries = atoi(optarg);
 492                                 if (maxretries < 0)
 493                                 {
 494                                         fprintf(stderr, "usage: pg_standby -r maxretries must be >= 0\n");
 495                                         usage();
 496                                         exit(2);
 497                                 }
 498                                 break;
 499                         case 's':                       /* Sleep time */
 500                                 sleeptime = atoi(optarg);
 501                                 if (sleeptime <= 0 || sleeptime > 60)
 502                                 {
 503                                         fprintf(stderr, "usage: pg_standby -s sleeptime incorrectly set\n");
 504                                         usage();
 505                                         exit(2);
 506                                 }
 507                                 break;
 508                         case 't':                       /* Trigger file */
 509                                 triggerPath = optarg;
 510                                 if (CheckForExternalTrigger())
 511                                         exit(1);        /* Normal exit, with non-zero */
 512                                 break;
 513                         case 'w':                       /* Max wait time */
 514                                 maxwaittime = atoi(optarg);
 515                                 if (maxwaittime < 0)
 516                                 {
 517                                         fprintf(stderr, "usage: pg_standby -w maxwaittime incorrectly set\n");
 518                                         usage();
 519                                         exit(2);
 520                                 }
 521                                 break;
 522                         default:
 523                                 usage();
 524                                 exit(2);
 525                                 break;
 526                 }
 527         }
 528
 529         /*
 530          * Parameter checking - after checking to see if trigger file present
 531          */
 532         if (argc == 1)
 533         {
 534                 usage();
 535                 exit(2);
 536         }
 537
 538         /*
 539          * We will go to the archiveLocation to get nextWALFileName.
 540          * nextWALFileName may not exist yet, which would not be an error, so we
 541          * separate the archiveLocation and nextWALFileName so we can check
 542          * separately whether archiveLocation exists, if not that is an error
 543          */
 544         if (optind < argc)
 545         {
 546                 archiveLocation = argv[optind];
 547                 optind++;
 548         }
 549         else
 550         {
 551                 fprintf(stderr, "pg_standby: must specify archiveLocation\n");
 552                 usage();
 553                 exit(2);
 554         }
 555
 556         if (optind < argc)
 557         {
 558                 nextWALFileName = argv[optind];
 559                 optind++;
 560         }
 561         else
 562         {
 563                 fprintf(stderr, "pg_standby: use %%f to specify nextWALFileName\n");
 564                 usage();
 565                 exit(2);
 566         }
 567
 568         if (optind < argc)
 569         {
 570                 xlogFilePath = argv[optind];
 571                 optind++;
 572         }
 573         else
 574         {
 575                 fprintf(stderr, "pg_standby: use %%p to specify xlogFilePath\n");
 576                 usage();
 577                 exit(2);
 578         }
 579
 580         if (optind < argc)
 581         {
 582                 restartWALFileName = argv[optind];
 583                 optind++;
 584         }
 585
 586         CustomizableInitialize();
 587
 588         need_cleanup = SetWALFileNameForCleanup();
 589
 590         if (debug)
 591         {
 592                 fprintf(stderr, "\nTrigger file                 : %s", triggerPath ? triggerPath : "<not set>");
 593                 fprintf(stderr, "\nWaiting for WAL file : %s", nextWALFileName);
 594                 fprintf(stderr, "\nWAL file path                : %s", WALFilePath);
 595                 fprintf(stderr, "\nRestoring to...              : %s", xlogFilePath);
 596                 fprintf(stderr, "\nSleep interval               : %d second%s",
 597                                 sleeptime, (sleeptime > 1 ? "s" : " "));
 598                 fprintf(stderr, "\nMax wait interval    : %d %s",
 599                                 maxwaittime, (maxwaittime > 0 ? "seconds" : "forever"));
 600                 fprintf(stderr, "\nCommand for restore  : %s", restoreCommand);
 601                 fprintf(stderr, "\nKeep archive history : ");
 602                 if (need_cleanup)
 603                         fprintf(stderr, "%s and later", exclusiveCleanupFileName);
 604                 else
 605                         fprintf(stderr, "No cleanup required");
 606                 fflush(stderr);
 607         }
 608
 609         /*
 610          * Check for initial history file: always the first file to be requested
 611          * It's OK if the file isn't there - all other files need to wait
 612          */
 613         if (strlen(nextWALFileName) > 8 &&
 614                 strspn(nextWALFileName, "0123456789ABCDEF") == 8 &&
 615                 strcmp(nextWALFileName + strlen(nextWALFileName) - strlen(".history"),
 616                            ".history") == 0)
 617         {
 618                 nextWALFileType = XLOG_HISTORY;
 619                 if (RestoreWALFileForRecovery())
 620                         exit(0);
 621                 else
 622                 {
 623                         if (debug)
 624                         {
 625                                 fprintf(stderr, "history file not found\n");
 626                                 fflush(stderr);
 627                         }
 628                         exit(1);
 629                 }
 630         }
 631
 632         /*
 633          * Main wait loop
 634          */
 635         while (!CustomizableNextWALFileReady() && !triggered)
 636         {
 637                 if (sleeptime <= 60)
 638                         pg_usleep(sleeptime * 1000000L);
 639
 640                 if (signaled)
 641                 {
 642                         triggered = true;
 643                         if (debug)
 644                         {
 645                                 fprintf(stderr, "\nsignaled to exit\n");
 646                                 fflush(stderr);
 647                         }
 648                 }
 649                 else
 650                 {
 651
 652                         if (debug)
 653                         {
 654                                 fprintf(stderr, "\nWAL file not present yet.");
 655                                 if (triggerPath)
 656                                         fprintf(stderr, " Checking for trigger file...");
 657                                 fflush(stderr);
 658                         }
 659
 660                         waittime += sleeptime;
 661
 662                         if (!triggered && (CheckForExternalTrigger() || (waittime >= maxwaittime && maxwaittime > 0)))
 663                         {
 664                                 triggered = true;
 665                                 if (debug && waittime >= maxwaittime && maxwaittime > 0)
 666                                         fprintf(stderr, "\nTimed out after %d seconds\n", waittime);
 667                         }
 668                 }
 669         }
 670
 671         /*
 672          * Action on exit
 673          */
 674         if (triggered)
 675                 exit(1);                                /* Normal exit, with non-zero */
 676
 677         /*
 678          * Once we have restored this file successfully we can remove some prior
 679          * WAL files. If this restore fails we musn't remove any file because some
 680          * of them will be requested again immediately after the failed restore,
 681          * or when we restart recovery.
 682          */
 683         if (RestoreWALFileForRecovery() && need_cleanup)
 684                 CustomizableCleanupPriorWALFiles();
 685
 686         return 0;
 687 }