3 list_packs.c -- list_packs utility to count Git packs and their objects
4 Copyright (C) 2016,2017 Kyle J. McKay.
7 This program is free software; you can redistribute it and/or
8 modify it under the terms of the GNU General Public License
9 as published by the Free Software Foundation; either version 2
10 of the License, or (at your option) any later version.
12 This program is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with this program; if not, write to the Free Software
19 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
24 This utility is intended to be used by a script to assist in determining
25 whether or not it's time to run gc in the case where gc.auto=0 and when
26 it is to provide a convenient mechanism to feed selected pack names to
27 a script for futher processing at gc time.
29 Various options are available to select which .pack files to inspect
30 including supplying the names. This utility is intended to be able
31 to read the pack names from the --export-pack-edges file that may be
32 produced by git fast-import without needing any preprocessing.
34 See the list_packs.txt file or run the command with no arguments for help.
37 #define _XOPEN_SOURCE 600
38 #undef _FILE_OFFSET_BITS
39 #define _FILE_OFFSET_BITS 64
45 #include <sys/types.h> /* in some cases required before dirent.h or sys/stat.h */
51 #include <arpa/inet.h>
58 #define PATH_BUFF 4096
60 #define PATH_BUFF PATH_MAX
63 static int fgetfn(FILE *f
, char *b
, size_t s
);
64 static void process_pack_filename(char *fn
, size_t bl
, int hasps
);
65 static void process_pack(const char *fn
, uint32_t objcnt
);
66 static void process_packs_finish(void);
69 struct pack_info
*next
;
75 struct ext_info
*next
;
76 size_t extlen
; /* strlen(ext) */
77 char ext
[12]; /* includes leading '.' and trailing '\0' */
78 int xxt
; /* >0 excludes if ext present, 0 excludes if ext NOT present */
81 #define MAX_EXT_LEN (sizeof(((const struct ext_info *)0)->ext) - 2)
82 #define MAX_SFX_LEN (sizeof(((const struct ext_info *)0)->ext) - 1)
83 #define BAD_EXT_CHARS ":./\\ \t\n\v\f\r"
86 #include "list_packs.inc"
90 static int opt_xix
= -1;
91 static long opt_limit
= 0;
92 static int opt_count
= 0;
93 static uint64_t count
= 0;
94 static uint64_t objlimit
= 0;
95 static int opt_desc
= 0;
96 static int opt_boundary
= 0;
97 static uint64_t maxlimit
= 0;
98 static uint64_t processed
= 0;
100 static struct pack_info
*packlist
= NULL
;
101 static size_t packcount
= 0;
103 static struct ext_info
*extlist
= NULL
;
104 static struct ext_info
*sfxlist
= NULL
;
106 static char fnbuff
[PATH_BUFF
];
108 static void die(const char *fmt
, ...)
114 vfprintf(stderr
, fmt
, args
);
119 static void dienomem(const char *what
)
123 die("list_packs: error: out of memory%s%s\n", (what
? " for " : ""),
127 static void dienopackmem(void)
129 dienomem("pack list");
132 static void dienoextmem(void)
134 dienomem("ext list");
137 static void dieusage(int err
)
139 FILE *f
= err
? stderr
: stdout
;
142 fprintf(f
, "%s", USAGE
);
146 static int has_suffix(const char *s
, size_t l
, const char *x
, size_t b
)
148 if (!s
|| !x
|| !b
|| l
< (b
+ 1) /* ?<suffix> */)
150 return !strncmp(s
+ l
- b
, x
, b
);
152 #define has_idx_suffix(s,l) has_suffix((s),(l),".idx",4)
153 #define has_pack_suffix(s,l) has_suffix((s),(l),".pack",5)
155 static int is_pack_sha1_name(const char *s
, size_t l
)
159 if (strncmp(s
, "pack-", 5) || strncmp(s
+ l
- 5, ".pack", 5))
161 return strspn(s
+ 5, "0123456789abcdefABCDEF") >= 40;
164 #define find_add_ext(ext) find_add_extsfx(&extlist, ext, '.')
165 #define find_add_sfx(sfx) find_add_extsfx(&sfxlist, sfx, 0)
166 static struct ext_info
*find_add_extsfx(struct ext_info
**list
, const char *extsfx
, char ch
)
168 size_t elen
= strlen(extsfx
);
169 size_t b
= ch
? 1 : 0;
170 struct ext_info
*result
= *list
;
172 while (result
&& strcmp(&result
->ext
[b
], extsfx
)) {
173 result
= result
->next
;
176 result
= (struct ext_info
*)malloc(sizeof(struct ext_info
));
179 result
->extlen
= elen
+ b
;
182 memcpy(&result
->ext
[b
], extsfx
, elen
+ 1);
183 if (elen
+ b
+ 1 < sizeof(result
->ext
))
184 memset(&result
->ext
[elen
+ b
+ 1], 0,
185 sizeof(result
->ext
) - (elen
+ b
+ 1));
187 result
->next
= *list
;
193 void handle_ext_option(const char *ext
, int v
)
196 struct ext_info
*einfo
;
198 if (!ext
|| !*ext
|| !(elen
= strlen(ext
)) ||
199 elen
> MAX_EXT_LEN
|| strcspn(ext
, BAD_EXT_CHARS
) != elen
)
200 dieusage(EXIT_FAILURE
);
201 if (!strcmp(ext
, "idx")) {
204 einfo
= find_add_ext(ext
);
209 void handle_sfx_option(const char *sfx
, int v
)
212 struct ext_info
*sinfo
;
214 if (!sfx
|| !*sfx
|| !(elen
= strlen(sfx
)) ||
215 elen
> MAX_SFX_LEN
|| strcspn(sfx
, BAD_EXT_CHARS
) != elen
||
216 strchr("0123456789abcdefABCDEF", sfx
[0]))
217 dieusage(EXIT_FAILURE
);
218 sinfo
= find_add_sfx(sfx
);
222 int main(int argc
, char *argv
[])
226 const char *only
= NULL
;
227 const char *dir
= NULL
;
231 for (argn
= 1; argn
< argc
; ++argn
) {
232 if (!strcmp(argv
[argn
], "-h") || !strcmp(argv
[argn
], "--help")) {
233 dieusage(EXIT_SUCCESS
);
234 } else if (!strcmp(argv
[argn
], "-q") || !strcmp(argv
[argn
], "--quiet")) {
236 } else if (!strcmp(argv
[argn
], "-a") || !strcmp(argv
[argn
], "--all")) {
238 } else if (!strcmp(argv
[argn
], "--exclude-idx")) {
240 } else if (!strcmp(argv
[argn
], "--exclude-no-idx")) {
242 } else if (!strcmp(argv
[argn
], "--exclude-keep")) {
243 handle_ext_option("keep", 1);
244 } else if (!strcmp(argv
[argn
], "--exclude-no-keep")) {
245 handle_ext_option("keep", 0);
246 } else if (!strcmp(argv
[argn
], "--exclude-bitmap")) {
247 handle_ext_option("bitmap", 1);
248 } else if (!strcmp(argv
[argn
], "--exclude-no-bitmap")) {
249 handle_ext_option("bitmap", 0);
250 } else if (!strcmp(argv
[argn
], "--exclude-bndl")) {
251 handle_ext_option("bndl", 1);
252 } else if (!strcmp(argv
[argn
], "--exclude-no-bndl")) {
253 handle_ext_option("bndl", 0);
254 } else if (!strcmp(argv
[argn
], "--count")) {
256 } else if (!strcmp(argv
[argn
], "--count-objects")) {
258 } else if (!strcmp(argv
[argn
], "--include-boundary")) {
260 } else if (!strcmp(argv
[argn
], "--exclude-ext")) {
262 dieusage(EXIT_FAILURE
);
263 handle_ext_option(argv
[argn
], 1);
264 } else if (!strcmp(argv
[argn
], "--exclude-no-ext")) {
266 dieusage(EXIT_FAILURE
);
267 handle_ext_option(argv
[argn
], 0);
268 } else if (!strcmp(argv
[argn
], "--exclude-sfx")) {
270 dieusage(EXIT_FAILURE
);
271 handle_sfx_option(argv
[argn
], 1);
272 } else if (!strcmp(argv
[argn
], "--exclude-no-sfx")) {
274 dieusage(EXIT_FAILURE
);
275 handle_sfx_option(argv
[argn
], 0);
276 } else if (!strcmp(argv
[argn
], "--exclude-limit")) {
281 dieusage(EXIT_FAILURE
);
282 limit
= strtol(argv
[argn
], &end
, 0);
283 if (!*argv
[argn
] || *end
|| !limit
)
284 dieusage(EXIT_FAILURE
);
286 } else if (!strcmp(argv
[argn
], "--object-limit")) {
291 dieusage(EXIT_FAILURE
);
292 limit
= strtol(argv
[argn
], &end
, 0);
293 if (!*argv
[argn
] || *end
|| !limit
)
294 dieusage(EXIT_FAILURE
);
297 objlimit
= (uint64_t)-limit
;
299 objlimit
= (uint64_t)limit
;
301 } else if (!strcmp(argv
[argn
], "--max-matches")) {
306 dieusage(EXIT_FAILURE
);
307 limit
= strtol(argv
[argn
], &end
, 0);
308 if (!*argv
[argn
] || *end
|| limit
<= 0)
309 dieusage(EXIT_FAILURE
);
310 maxlimit
= (uint64_t)limit
;
311 } else if (!strcmp(argv
[argn
], "--only")) {
312 if (++argn
>= argc
|| !*argv
[argn
])
313 dieusage(EXIT_FAILURE
);
315 } else if (!strcmp(argv
[argn
], "-C")) {
316 if (++argn
>= argc
|| !*argv
[argn
])
317 dieusage(EXIT_FAILURE
);
318 if (chdir(argv
[argn
])) {
320 fprintf(stderr
, "list_packs: error: "
321 "chdir '%s' failed\n", argv
[argn
]);
324 } else if (!strcmp(argv
[argn
], "--")) {
327 } else if (argv
[argn
][0] == '-' && argv
[argn
][1]) {
328 dieusage(EXIT_FAILURE
);
333 if (argn
< argc
&& *argv
[argn
])
335 if (argn
!= argc
|| (!only
&& !dir
) || (only
&& dir
) || (only
&& opt_a
))
336 dieusage(EXIT_FAILURE
);
338 if (!strcmp(only
, "-")) {
342 in
= fopen(only
, "r");
344 die("list_packs: error: could not open %s\n", only
);
346 while (fgetfn(in
, fnbuff
, sizeof(fnbuff
) - (MAX_EXT_LEN
+ 1))) {
348 size_t l
= strlen(fn
);
353 if (l
> 2 && !strncmp(fn
, "./", 2)) {
357 ips
= has_pack_suffix(fn
, l
);
358 process_pack_filename(fn
, (ips
? l
- 5 : l
), ips
);
368 while (l
> 1 && dir
[l
-1] == '/') {
371 if (l
> 2 && !strncmp(dir
, "./", 2)) {
375 if (l
+ 10 /* "/?.bitmap\0" */ > PATH_BUFF
)
376 die("list_packs: error: dirname too long\n");
377 memcpy(fnbuff
, dir
, l
);
381 die("list_packs: error: could not read directory %s\n", fnbuff
);
382 if (!strcmp(fnbuff
, ".")) {
386 if (l
&& fnbuff
[l
-1] != '/')
388 while ((e
= readdir(d
)) != NULL
) {
389 /* d_namlen is a nice, but non-POSIX extension */
390 size_t el
= strlen(e
->d_name
);
392 if (has_pack_suffix(e
->d_name
, el
) &&
393 (opt_a
|| is_pack_sha1_name(e
->d_name
, el
))) {
394 if (l
+ el
+ 3 /* "ap\0" */ > PATH_BUFF
) {
396 fprintf(stderr
, "list_packs: warning: "
397 "ignored input filename greater "
398 "than %d characters long\n",
402 memcpy(fnbuff
+ l
, e
->d_name
, el
+ 1 /* \0 */);
403 process_pack_filename(fnbuff
, l
+ el
- 5 /* .pack */, 1);
408 process_packs_finish();
410 printf("%"PRIu64
"\n", count
);
415 #define FNDELIM "\t\n\v\f\r :"
417 static int fgetfn(FILE *f
, char *b
, size_t s
)
422 if (!fgets(b
, (int)s
, f
)) {
425 fprintf(stderr
, "list_packs: error: an error "
426 "occurred reading pack name list file\n");
434 fnl
= strcspn(b
, FNDELIM
);
435 if (b
[l
-1] != '\n' && !feof(f
)) {
438 while ((ch
= getc_unlocked(f
)) != EOF
&& ch
!= '\n') {
446 if (fnl
< l
|| (!ferror(f
) && !trunc
)) {
452 fprintf(stderr
, "list_packs: error: an error "
453 "occurred reading pack name list file\n");
457 fprintf(stderr
, "list_packs: warning: ignored input filename "
458 "greater than %d characters long\n", (int)s
- 2);
463 static int file_exists(const char *fn
, struct stat
*s
)
466 if (S_ISREG(s
->st_mode
))
469 fprintf(stderr
, "list_packs: warning: ignoring "
470 "non-file '%s'\n", fn
);
475 static void process_pack_filename(char *fn
, size_t bl
, int hasps
)
485 const struct ext_info
*einfo
;
488 if (stat(fn
, &ps
) || !S_ISREG(ps
.st_mode
)) {
490 fprintf(stderr
, "list_packs: warning: ignoring "
491 "non-file '%s'\n", fn
);
494 if (ps
.st_size
< 32) {
496 fprintf(stderr
, "list_packs: warning: ignoring "
497 "invalid pack file '%s'\n", fn
);
503 if (einfo
->xxt
>= 0) {
506 hsfx
= (bl
>= einfo
->extlen
) &&
507 !strncmp(fn
+ bl
- einfo
->extlen
, einfo
->ext
, einfo
->extlen
);
511 } else if ((sfxor
= hsfx
)) {
521 if (einfo
->xxt
>= 0) {
524 memcpy(fn
+ bl
, einfo
->ext
, einfo
->extlen
+ 1);
525 hext
= file_exists(fn
, &es
);
526 if ((einfo
->xxt
&& hext
) || (!einfo
->xxt
&& !hext
))
534 memcpy(fn
+ bl
, ".idx", 5);
535 hx
= file_exists(fn
, &es
);
536 if ((opt_xix
&& hx
) || (!opt_xix
&& !hx
))
540 memcpy(fn
+ bl
, ".pack", 6);
546 fprintf(stderr
, "list_packs: warning: ignoring "
547 "unopenable pack file '%s'\n", fn
);
550 if (fread(&hdr
, 12, 1, f
) != 1) {
553 fprintf(stderr
, "list_packs: warning: ignoring "
554 "unreadable pack file '%s'\n", fn
);
558 packver
= ntohl(hdr
.u
[1]);
559 objcnt
= ntohl(hdr
.u
[2]);
560 if (memcmp(hdr
.c
, "PACK", 4) || (packver
!= 2 && packver
!= 3) ||
561 ps
.st_size
< ((off_t
)objcnt
+ 32)) {
563 fprintf(stderr
, "list_packs: warning: ignoring "
564 "invalid pack file '%s'\n", fn
);
567 if (!opt_xix
&& es
.st_size
< ((off_t
)objcnt
* 28 + 1072)) {
569 fprintf(stderr
, "list_packs: warning: ignoring pack "
570 "with invalid idx file '%.*s.idx'\n", (int)bl
, fn
);
574 if ((opt_limit
> 0 && objcnt
>= (uint32_t)opt_limit
) ||
575 (opt_limit
< 0 && objcnt
< (uint32_t)-opt_limit
))
578 /* the PACK file passed all the checks, process it */
580 size_t fnlen
= strlen(fn
);
581 struct pack_info
*info
= (struct pack_info
*)
582 malloc(sizeof(struct pack_info
) + fnlen
);
586 info
->objcount
= objcnt
;
587 memcpy(info
->filename
, fn
, fnlen
+ 1);
588 info
->next
= packlist
;
592 process_pack(fn
, objcnt
);
596 static void process_pack(const char *fn
, uint32_t objcnt
)
598 if (maxlimit
&& processed
>= maxlimit
)
611 static void process_pack_info(const struct pack_info
*pack
)
613 process_pack(pack
->filename
, pack
->objcount
);
616 static int sort_asc(const void *p
, const void *q
)
618 const struct pack_info
**a
= (const struct pack_info
**)p
;
619 const struct pack_info
**b
= (const struct pack_info
**)q
;
620 if ((*a
)->objcount
< (*b
)->objcount
)
622 if ((*a
)->objcount
> (*b
)->objcount
)
624 return strcmp((*a
)->filename
, (*b
)->filename
);
627 static int sort_dsc(const void *p
, const void *q
)
629 return sort_asc(q
, p
);
632 static void process_packs_finish(void)
634 struct pack_info
**table
, *p
;
638 if (!objlimit
|| !packlist
|| !packcount
)
640 table
= (struct pack_info
**)malloc(sizeof(struct pack_info
*) * packcount
);
649 qsort(table
, packcount
, sizeof(struct pack_info
*), (opt_desc
? sort_dsc
: sort_asc
));
651 for (i
=0; i
< packcount
; ++i
) {
652 tally
+= table
[i
]->objcount
;
653 if (tally
<= objlimit
) {
654 process_pack_info(table
[i
]);
657 process_pack_info(table
[i
]);