src/list_packs.c

   1 /*
   2
   3 list_packs.c -- list_packs utility to count Git packs and their objects
   4 Copyright (C) 2016 Kyle J. McKay.  All rights reserved.
   5
   6 This program is free software; you can redistribute it and/or
   7 modify it under the terms of the GNU General Public License
   8 as published by the Free Software Foundation; either version 2
   9 of the License, or (at your option) any later version.
  10
  11 This program is distributed in the hope that it will be useful,
  12 but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 GNU General Public License for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with this program; if not, write to the Free Software
  18 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  19
  20 */
  21
  22 /*
  23   This utility is intended to be used by a script to assist in determining
  24   whether or not it's time to run gc in the case where gc.auto=0 and when
  25   it is to provide a convenient mechanism to feed selected pack names to
  26   a script for futher processing at gc time.
  27
  28   Various options are available to select which .pack files to inspect
  29   including supplying the names.  This utility is intended to be able
  30   to read the pack names from the --export-pack-edges file that may be
  31   produced by git fast-import without needing any preprocessing.
  32
  33   See the list_packs.txt file or run the command with no arguments for help.
  34 */
  35
  36 #define _XOPEN_SOURCE 600
  37 #undef _FILE_OFFSET_BITS
  38 #define _FILE_OFFSET_BITS 64
  39 #include <stdarg.h>
  40 #include <stddef.h>
  41 #include <stdint.h>
  42 #include <stdio.h>
  43 #include <stdlib.h>
  44 #include <sys/types.h> /* in some cases required before dirent.h or sys/stat.h */
  45 #include <dirent.h>
  46 #include <inttypes.h>
  47 #include <limits.h>
  48 #include <string.h>
  49 #include <unistd.h>
  50 #include <arpa/inet.h>
  51 #include <sys/stat.h>
  52
  53 #ifndef PATH_MAX
  54 #define PATH_MAX 1024
  55 #endif
  56 #if PATH_MAX < 4096
  57 #define PATH_BUFF 4096
  58 #else
  59 #define PATH_BUFF PATH_MAX
  60 #endif
  61
  62 static int fgetfn(FILE *f, char *b, size_t s);
  63 static void process_pack_filename(char *fn, size_t bl, int hasps);
  64
  65 const char USAGE[] =
  66 #include "list_packs.inc"
  67 ;
  68
  69 static int opt_q = 0;
  70 static int opt_xkp = -1;
  71 static int opt_xbt = -1;
  72 static int opt_xbn = -1;
  73 static int opt_xix = -1;
  74 static long opt_limit = 0;
  75 static int opt_count = 0;
  76 static uint64_t count = 0;
  77
  78 static char fnbuff[PATH_BUFF];
  79
  80 static void die(const char *fmt, ...)
  81 {
  82         va_list args;
  83         va_start(args, fmt);
  84         fflush(stdout);
  85         if (!opt_q)
  86                 vfprintf(stderr, fmt, args);
  87         va_end(args);
  88         exit(EXIT_FAILURE);
  89 }
  90
  91 static void dieusage(int err)
  92 {
  93         FILE *f = err ? stderr : stdout;
  94         fflush(stdout);
  95         if (!err || !opt_q)
  96                 fprintf(f, "%s", USAGE);
  97         exit(err);
  98 }
  99
 100 static int has_suffix(const char *s, size_t l, const char *x, size_t b)
 101 {
 102         if (!s || !x || !b || l < (b + 1) /* ?<suffix> */)
 103                 return 0;
 104         return !strncmp(s + l - b, x, b);
 105 }
 106 #define has_idx_suffix(s,l) has_suffix((s),(l),".idx",4)
 107 #define has_bndl_suffix(s,l) has_suffix((s),(l),".bndl",5)
 108 #define has_keep_suffix(s,l) has_suffix((s),(l),".keep",5)
 109 #define has_pack_suffix(s,l) has_suffix((s),(l),".pack",5)
 110 #define has_bitmap_suffix(s,l) has_suffix((s),(l),".bitmap",7)
 111
 112 static int is_pack_sha1_name(const char *s, size_t l)
 113 {
 114         if (!s || l != 50)
 115                 return 0;
 116         if (strncmp(s, "pack-", 5) || strncmp(s + l - 5, ".pack", 5))
 117                 return 0;
 118         return strspn(s + 5, "0123456789abcdefABCDEF") == 40;
 119 }
 120
 121 int main(int argc, char *argv[])
 122 {
 123         int argn;
 124         int opt_a = 0;
 125         const char *only = NULL;
 126         const char *dir = NULL;
 127         int is_stdin = 0;
 128         FILE *in = NULL;
 129
 130         for (argn = 1; argn < argc; ++argn) {
 131                 if (!strcmp(argv[argn], "-h") || !strcmp(argv[argn], "--help")) {
 132                         dieusage(EXIT_SUCCESS);
 133                 } else if (!strcmp(argv[argn], "-q") || !strcmp(argv[argn], "--quiet")) {
 134                         opt_q = 1;
 135                 } else if (!strcmp(argv[argn], "-a") || !strcmp(argv[argn], "--all")) {
 136                         opt_a = 1;
 137                 } else if (!strcmp(argv[argn], "--exclude-keep")) {
 138                         opt_xkp = 1;
 139                 } else if (!strcmp(argv[argn], "--exclude-no-keep")) {
 140                         opt_xkp = 0;
 141                 } else if (!strcmp(argv[argn], "--exclude-bitmap")) {
 142                         opt_xbt = 1;
 143                 } else if (!strcmp(argv[argn], "--exclude-no-bitmap")) {
 144                         opt_xbt = 0;
 145                 } else if (!strcmp(argv[argn], "--exclude-bndl")) {
 146                         opt_xbn = 1;
 147                 } else if (!strcmp(argv[argn], "--exclude-no-bndl")) {
 148                         opt_xbn = 0;
 149                 } else if (!strcmp(argv[argn], "--exclude-idx")) {
 150                         opt_xix = 1;
 151                 } else if (!strcmp(argv[argn], "--exclude-no-idx")) {
 152                         opt_xix = 0;
 153                 } else if (!strcmp(argv[argn], "--count")) {
 154                         opt_count = 1;
 155                 } else if (!strcmp(argv[argn], "--count-objects")) {
 156                         opt_count = 2;
 157                 } else if (!strcmp(argv[argn], "--exclude-limit")) {
 158                         char *end;
 159                         long limit = 0;
 160
 161                         if (++argn >= argc)
 162                                 dieusage(EXIT_FAILURE);
 163                         limit = strtol(argv[argn], &end, 0);
 164                         if (!*argv[argn] || *end || !limit)
 165                                 dieusage(EXIT_FAILURE);
 166                         opt_limit = limit;
 167                 } else if (!strcmp(argv[argn], "--only")) {
 168                         if (++argn >= argc || !*argv[argn])
 169                                 dieusage(EXIT_FAILURE);
 170                         only = argv[argn];
 171                 } else if (!strcmp(argv[argn], "-C")) {
 172                         if (++argn >= argc || !*argv[argn])
 173                                 dieusage(EXIT_FAILURE);
 174                         if (chdir(argv[argn])) {
 175                                 if (!opt_q)
 176                                         fprintf(stderr, "list_packs: error: "
 177                                                 "chdir '%s' failed\n", argv[argn]);
 178                                 exit(EXIT_FAILURE);
 179                         }
 180                 } else if (!strcmp(argv[argn], "--")) {
 181                         ++argn;
 182                         break;
 183                 } else if (argv[argn][0] == '-' && argv[argn][1]) {
 184                         dieusage(EXIT_FAILURE);
 185                 } else {
 186                         break;
 187                 }
 188         }
 189         if (argn < argc && *argv[argn])
 190                 dir = argv[argn++];
 191         if (argn != argc || (!only && !dir) || (only && dir) || (only && opt_a))
 192                 dieusage(EXIT_FAILURE);
 193         if (only) {
 194                 if (!strcmp(only, "-")) {
 195                         is_stdin = 1;
 196                         in = stdin;
 197                 } else {
 198                         in = fopen(only, "r");
 199                         if (!in)
 200                                 die("list_packs: error: could not open %s\n", only);
 201                 }
 202                 while (fgetfn(in, fnbuff, sizeof(fnbuff) - 7 /* .bitmap */)) {
 203                         char *fn = fnbuff;
 204                         size_t l = strlen(fn);
 205                         int ips;
 206
 207                         if (!l)
 208                                 continue;
 209                         if (l > 2 && !strncmp(fn, "./", 2)) {
 210                                 fn += 2;
 211                                 l -= 2;
 212                         }
 213                         ips = has_pack_suffix(fn, l);
 214                         process_pack_filename(fn, (ips ? l - 5 : l), ips);
 215                 }
 216                 if (!is_stdin)
 217                         fclose(in);
 218         } else {
 219                 size_t l;
 220                 DIR *d;
 221                 struct dirent *e;
 222
 223                 l = strlen(dir);
 224                 while (l > 1 && dir[l-1] == '/') {
 225                         --l;
 226                 }
 227                 if (l > 2 && !strncmp(dir, "./", 2)) {
 228                         dir += 2;
 229                         l -= 2;
 230                 }
 231                 if (l + 10 /* "/?.bitmap\0" */ > PATH_BUFF)
 232                         die("list_packs: error: dirname too long\n");
 233                 memcpy(fnbuff, dir, l);
 234                 fnbuff[l] = '\0';
 235                 d = opendir(fnbuff);
 236                 if (!d)
 237                         die("list_packs: error: could not read directory %s\n", fnbuff);
 238                 if (!strcmp(fnbuff, ".")) {
 239                         l = 0;
 240                         fnbuff[0] = '\0';
 241                 }
 242                 if (l && fnbuff[l-1] != '/')
 243                         fnbuff[l++] = '/';
 244                 while ((e = readdir(d)) != NULL) {
 245                         /* d_namlen is a nice, but non-POSIX extension */
 246                         size_t el = strlen(e->d_name);
 247
 248                         if (has_pack_suffix(e->d_name, el) &&
 249                             (opt_a || is_pack_sha1_name(e->d_name, el))) {
 250                                 if (l + el + 3 /* "ap\0" */ > PATH_BUFF) {
 251                                         if (!opt_q)
 252                                                 fprintf(stderr, "list_packs: warning: "
 253                                                         "ignored input filename greater "
 254                                                         "than %d characters long\n",
 255                                                         PATH_BUFF - 3);
 256                                         continue;
 257                                 }
 258                                 memcpy(fnbuff + l, e->d_name, el + 1 /* \0 */);
 259                                 process_pack_filename(fnbuff, l + el - 5 /* .pack */, 1);
 260                         }
 261                 }
 262                 closedir(d);
 263         }
 264         if (opt_count)
 265                 printf("%"PRIu64"\n", count);
 266
 267         return EXIT_SUCCESS;
 268 }
 269
 270 #define FNDELIM "\t\n\v\f\r :"
 271
 272 static int fgetfn(FILE *f, char *b, size_t s)
 273 {
 274         size_t l, fnl;
 275         int trunc;
 276
 277         if (!fgets(b, (int)s, f)) {
 278                 if (ferror(f)) {
 279                         if (!opt_q)
 280                                 fprintf(stderr, "list_packs: error: an error "
 281                                         "occurred reading pack name list file\n");
 282                         exit(EXIT_FAILURE);
 283                 }
 284                 return 0;
 285         }
 286         if (!*b)
 287                 return 1;
 288         l = strlen(b);
 289         fnl = strcspn(b, FNDELIM);
 290         if (b[l-1] != '\n' && !feof(f)) {
 291                 int ch;
 292                 flockfile(f);
 293                 while ((ch = getc_unlocked(f)) != EOF && ch != '\n') {
 294                         /* loop */
 295                 }
 296                 funlockfile(f);
 297                 trunc = 1;
 298         } else {
 299                 trunc = 0;
 300         }
 301         if (fnl < l || (!ferror(f) && !trunc)) {
 302                 b[fnl] = '\0';
 303                 return 1;
 304         }
 305         if (ferror(f)) {
 306                 if (!opt_q)
 307                         fprintf(stderr, "list_packs: error: an error "
 308                                 "occurred reading pack name list file\n");
 309                 exit(EXIT_FAILURE);
 310         }
 311         if (!opt_q)
 312                 fprintf(stderr, "list_packs: warning: ignored input filename "
 313                         "greater than %d characters long\n", (int)s - 2);
 314         *b = '\0';
 315         return 1;
 316 }
 317
 318 static int file_exists(const char *fn, struct stat *s)
 319 {
 320         if (!stat(fn, s)) {
 321                 if (S_ISREG(s->st_mode))
 322                         return 1;
 323                 if (!opt_q)
 324                         fprintf(stderr, "list_packs: warning: ignoring "
 325                                 "non-file '%s'\n", fn);
 326         }
 327         return 0;
 328 }
 329
 330 static void process_pack_filename(char *fn, size_t bl, int hasps)
 331 {
 332         struct stat ps, es;
 333         FILE *f;
 334         union {
 335                 uint32_t u[3];
 336                 char c[12];
 337         } hdr;
 338         uint32_t packver;
 339         uint32_t objcnt;
 340
 341         if (stat(fn, &ps) || !S_ISREG(ps.st_mode)) {
 342                 if (!opt_q)
 343                         fprintf(stderr, "list_packs: warning: ignoring "
 344                                 "non-file '%s'\n", fn);
 345                 return;
 346         }
 347         if (ps.st_size < 32) {
 348                 if (!opt_q)
 349                         fprintf(stderr, "list_packs: warning: ignoring "
 350                                 "invalid pack file '%s'\n", fn);
 351                 return;
 352         }
 353         if (opt_xkp >= 0) {
 354                 int hk;
 355
 356                 memcpy(fn + bl, ".keep", 6);
 357                 hk = file_exists(fn, &es);
 358                 if ((opt_xkp && hk) || (!opt_xkp && !hk))
 359                         return;
 360         }
 361         if (opt_xbt >= 0) {
 362                 int hb;
 363
 364                 memcpy(fn + bl, ".bitmap", 8);
 365                 hb = file_exists(fn, &es);
 366                 if ((opt_xbt && hb) || (!opt_xbt && !hb))
 367                         return;
 368         }
 369         if (opt_xbn >= 0) {
 370                 int hn;
 371
 372                 memcpy(fn + bl, ".bndl", 6);
 373                 hn = file_exists(fn, &es);
 374                 if ((opt_xbn && hn) || (!opt_xbn && !hn))
 375                         return;
 376         }
 377         if (opt_xix >= 0) {
 378                 int hx;
 379
 380                 memcpy(fn + bl, ".idx", 5);
 381                 hx = file_exists(fn, &es);
 382                 if ((opt_xix && hx) || (!opt_xix && !hx))
 383                         return;
 384         }
 385         if (hasps)
 386                 memcpy(fn + bl, ".pack", 6);
 387         else
 388                 fn[bl] = '\0';
 389         f = fopen(fn, "rb");
 390         if (!f) {
 391                 if (!opt_q)
 392                         fprintf(stderr, "list_packs: warning: ignoring "
 393                                 "unopenable pack file '%s'\n", fn);
 394                 return;
 395         }
 396         if (fread(&hdr, 12, 1, f) != 1) {
 397                 fclose(f);
 398                 if (!opt_q)
 399                         fprintf(stderr, "list_packs: warning: ignoring "
 400                                 "unreadable pack file '%s'\n", fn);
 401                 return;
 402         }
 403         fclose(f);
 404         packver = ntohl(hdr.u[1]);
 405         objcnt = ntohl(hdr.u[2]);
 406         if (memcmp(hdr.c, "PACK", 4) || (packver != 2 && packver != 3) ||
 407             ps.st_size < ((off_t)objcnt + 32)) {
 408                 if (!opt_q)
 409                         fprintf(stderr, "list_packs: warning: ignoring "
 410                                 "invalid pack file '%s'\n", fn);
 411                 return;
 412         }
 413         if (!opt_xix && es.st_size < ((off_t)objcnt * 28 + 1072)) {
 414                 if (!opt_q)
 415                         fprintf(stderr, "list_packs: warning: ignoring pack "
 416                                 "with invalid idx file '%.*s.idx'\n", (int)bl, fn);
 417                 return;
 418         }
 419         if (opt_limit) {
 420                 if ((opt_limit > 0 && objcnt >= (uint32_t)opt_limit) ||
 421                     (opt_limit < 0 && objcnt < (uint32_t)-opt_limit))
 422                         return;
 423         }
 424         /* the PACK file passed all the checks, process it */
 425         if (opt_count) {
 426                 if (opt_count > 1)
 427                         count += objcnt;
 428                 else
 429                         ++count;
 430         } else {
 431                 printf("%s\n", fn);
 432         }
 433 }