2 * Copyright (c) 2020 Stefan Sperling <stsp@openbsd.org>
4 * Permission to use, copy, modify, and distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 #include <sys/types.h>
20 #include <sys/socket.h>
33 #include "got_error.h"
34 #include "got_cancel.h"
35 #include "got_object.h"
36 #include "got_reference.h"
37 #include "got_repository.h"
38 #include "got_repository_admin.h"
39 #include "got_opentemp.h"
42 #include "got_lib_delta.h"
43 #include "got_lib_object.h"
44 #include "got_lib_object_idset.h"
45 #include "got_lib_object_cache.h"
46 #include "got_lib_pack.h"
47 #include "got_lib_privsep.h"
48 #include "got_lib_repository.h"
49 #include "got_lib_pack_create.h"
50 #include "got_lib_sha1.h"
51 #include "got_lib_lockfile.h"
54 #define nitems(_a) (sizeof((_a)) / sizeof((_a)[0]))
57 static const struct got_error
*
58 get_reflist_object_ids(struct got_object_id
***ids
, int *nobjects
,
59 unsigned int wanted_obj_type_mask
, struct got_reflist_head
*refs
,
60 struct got_repository
*repo
,
61 got_cancel_cb cancel_cb
, void *cancel_arg
)
63 const struct got_error
*err
= NULL
;
64 const size_t alloc_chunksz
= 256;
66 struct got_reflist_entry
*re
;
72 *ids
= reallocarray(NULL
, alloc_chunksz
, sizeof(struct got_object_id
*));
74 return got_error_from_errno("reallocarray");
75 nalloc
= alloc_chunksz
;
77 TAILQ_FOREACH(re
, refs
, entry
) {
78 struct got_object_id
*id
;
81 err
= cancel_cb(cancel_arg
);
86 err
= got_ref_resolve(&id
, repo
, re
->ref
);
90 if (wanted_obj_type_mask
!= GOT_OBJ_TYPE_ANY
) {
92 err
= got_object_get_type(&obj_type
, repo
, id
);
95 if ((wanted_obj_type_mask
& (1 << obj_type
)) == 0) {
102 if (nalloc
<= *nobjects
) {
103 struct got_object_id
**new;
104 new = recallocarray(*ids
, nalloc
,
105 nalloc
+ alloc_chunksz
,
106 sizeof(struct got_object_id
*));
108 err
= got_error_from_errno(
113 nalloc
+= alloc_chunksz
;
115 (*ids
)[*nobjects
] = id
;
116 if ((*ids
)[*nobjects
] == NULL
) {
117 err
= got_error_from_errno("got_object_id_dup");
124 for (i
= 0; i
< *nobjects
; i
++)
133 const struct got_error
*
134 got_repo_pack_objects(FILE **packfile
, struct got_object_id
**pack_hash
,
135 struct got_reflist_head
*include_refs
,
136 struct got_reflist_head
*exclude_refs
, struct got_repository
*repo
,
137 int loose_obj_only
, got_pack_progress_cb progress_cb
, void *progress_arg
,
138 got_cancel_cb cancel_cb
, void *cancel_arg
)
140 const struct got_error
*err
= NULL
;
141 struct got_object_id
**ours
= NULL
, **theirs
= NULL
;
142 int nours
= 0, ntheirs
= 0, packfd
= -1, i
;
143 char *tmpfile_path
= NULL
, *path
= NULL
, *packfile_path
= NULL
;
144 char *sha1_str
= NULL
;
149 if (asprintf(&path
, "%s/%s/packing.pack",
150 got_repo_get_path_git_dir(repo
), GOT_OBJECTS_PACK_DIR
) == -1) {
151 err
= got_error_from_errno("asprintf");
154 err
= got_opentemp_named_fd(&tmpfile_path
, &packfd
, path
);
158 if (fchmod(packfd
, GOT_DEFAULT_FILE_MODE
) != 0) {
159 err
= got_error_from_errno2("fchmod", tmpfile_path
);
163 *packfile
= fdopen(packfd
, "w");
164 if (*packfile
== NULL
) {
165 err
= got_error_from_errno2("fdopen", tmpfile_path
);
170 err
= get_reflist_object_ids(&ours
, &nours
,
171 (1 << GOT_OBJ_TYPE_COMMIT
) | (1 << GOT_OBJ_TYPE_TAG
),
172 include_refs
, repo
, cancel_cb
, cancel_arg
);
177 err
= got_error(GOT_ERR_CANNOT_PACK
);
181 if (!TAILQ_EMPTY(exclude_refs
)) {
182 err
= get_reflist_object_ids(&theirs
, &ntheirs
,
183 (1 << GOT_OBJ_TYPE_COMMIT
) | (1 << GOT_OBJ_TYPE_TAG
),
185 cancel_cb
, cancel_arg
);
190 *pack_hash
= calloc(1, sizeof(**pack_hash
));
191 if (*pack_hash
== NULL
) {
192 err
= got_error_from_errno("calloc");
196 err
= got_pack_create((*pack_hash
)->sha1
, *packfile
, theirs
, ntheirs
,
197 ours
, nours
, repo
, loose_obj_only
, 0, progress_cb
, progress_arg
,
198 cancel_cb
, cancel_arg
);
202 err
= got_object_id_str(&sha1_str
, *pack_hash
);
205 if (asprintf(&packfile_path
, "%s/%s/pack-%s.pack",
206 got_repo_get_path_git_dir(repo
), GOT_OBJECTS_PACK_DIR
,
208 err
= got_error_from_errno("asprintf");
212 if (fflush(*packfile
) == -1) {
213 err
= got_error_from_errno("fflush");
216 if (fseek(*packfile
, 0L, SEEK_SET
) == -1) {
217 err
= got_error_from_errno("fseek");
220 if (rename(tmpfile_path
, packfile_path
) == -1) {
221 err
= got_error_from_errno3("rename", tmpfile_path
,
228 for (i
= 0; i
< nours
; i
++)
231 for (i
= 0; i
< ntheirs
; i
++)
234 if (packfd
!= -1 && close(packfd
) == -1 && err
== NULL
)
235 err
= got_error_from_errno2("close", packfile_path
);
236 if (tmpfile_path
&& unlink(tmpfile_path
) == -1 && err
== NULL
)
237 err
= got_error_from_errno2("unlink", tmpfile_path
);
252 const struct got_error
*
253 got_repo_index_pack(FILE *packfile
, struct got_object_id
*pack_hash
,
254 struct got_repository
*repo
,
255 got_pack_index_progress_cb progress_cb
, void *progress_arg
,
256 got_cancel_cb cancel_cb
, void *cancel_arg
)
261 int npackfd
= -1, idxfd
= -1, nidxfd
= -1;
263 int idxstatus
, done
= 0;
264 const struct got_error
*err
;
265 struct imsgbuf idxibuf
;
267 char *tmpidxpath
= NULL
;
268 char *packfile_path
= NULL
, *idxpath
= NULL
, *id_str
= NULL
;
269 const char *repo_path
= got_repo_get_path_git_dir(repo
);
272 for (i
= 0; i
< nitems(tmpfds
); i
++)
275 if (asprintf(&path
, "%s/%s/indexing.idx",
276 repo_path
, GOT_OBJECTS_PACK_DIR
) == -1) {
277 err
= got_error_from_errno("asprintf");
280 err
= got_opentemp_named_fd(&tmpidxpath
, &idxfd
, path
);
284 if (fchmod(idxfd
, GOT_DEFAULT_FILE_MODE
) != 0) {
285 err
= got_error_from_errno2("fchmod", tmpidxpath
);
291 err
= got_error_from_errno("dup");
295 for (i
= 0; i
< nitems(tmpfds
); i
++) {
296 tmpfds
[i
] = got_opentempfd();
297 if (tmpfds
[i
] == -1) {
298 err
= got_error_from_errno("got_opentempfd");
303 err
= got_object_id_str(&id_str
, pack_hash
);
307 if (asprintf(&packfile_path
, "%s/%s/pack-%s.pack",
308 repo_path
, GOT_OBJECTS_PACK_DIR
, id_str
) == -1) {
309 err
= got_error_from_errno("asprintf");
313 if (fstat(fileno(packfile
), &sb
) == -1) {
314 err
= got_error_from_errno2("fstat", packfile_path
);
318 if (asprintf(&idxpath
, "%s/%s/pack-%s.idx",
319 repo_path
, GOT_OBJECTS_PACK_DIR
, id_str
) == -1) {
320 err
= got_error_from_errno("asprintf");
324 if (socketpair(AF_UNIX
, SOCK_STREAM
, PF_UNSPEC
, imsg_idxfds
) == -1) {
325 err
= got_error_from_errno("socketpair");
330 err
= got_error_from_errno("fork");
332 } else if (idxpid
== 0)
333 got_privsep_exec_child(imsg_idxfds
,
334 GOT_PATH_PROG_INDEX_PACK
, packfile_path
);
335 if (close(imsg_idxfds
[1]) == -1) {
336 err
= got_error_from_errno("close");
339 imsg_init(&idxibuf
, imsg_idxfds
[0]);
341 npackfd
= dup(fileno(packfile
));
343 err
= got_error_from_errno("dup");
346 err
= got_privsep_send_index_pack_req(&idxibuf
, pack_hash
->sha1
,
351 err
= got_privsep_send_index_pack_outfd(&idxibuf
, nidxfd
);
355 for (i
= 0; i
< nitems(tmpfds
); i
++) {
356 err
= got_privsep_send_tmpfd(&idxibuf
, tmpfds
[i
]);
363 int nobj_total
, nobj_indexed
, nobj_loose
, nobj_resolved
;
366 err
= cancel_cb(cancel_arg
);
371 err
= got_privsep_recv_index_progress(&done
, &nobj_total
,
372 &nobj_indexed
, &nobj_loose
, &nobj_resolved
,
376 if (nobj_indexed
!= 0) {
377 err
= progress_cb(progress_arg
, sb
.st_size
,
378 nobj_total
, nobj_indexed
, nobj_loose
,
383 imsg_clear(&idxibuf
);
385 if (close(imsg_idxfds
[0]) == -1) {
386 err
= got_error_from_errno("close");
389 if (waitpid(idxpid
, &idxstatus
, 0) == -1) {
390 err
= got_error_from_errno("waitpid");
394 if (rename(tmpidxpath
, idxpath
) == -1) {
395 err
= got_error_from_errno3("rename", tmpidxpath
, idxpath
);
402 if (tmpidxpath
&& unlink(tmpidxpath
) == -1 && err
== NULL
)
403 err
= got_error_from_errno2("unlink", tmpidxpath
);
404 if (npackfd
!= -1 && close(npackfd
) == -1 && err
== NULL
)
405 err
= got_error_from_errno("close");
406 if (idxfd
!= -1 && close(idxfd
) == -1 && err
== NULL
)
407 err
= got_error_from_errno("close");
408 for (i
= 0; i
< nitems(tmpfds
); i
++) {
409 if (tmpfds
[i
] != -1 && close(tmpfds
[i
]) == -1 && err
== NULL
)
410 err
= got_error_from_errno("close");
418 const struct got_error
*
419 got_repo_find_pack(FILE **packfile
, struct got_object_id
**pack_hash
,
420 struct got_repository
*repo
, const char *packfile_path
)
422 const struct got_error
*err
= NULL
;
423 const char *packdir_path
= NULL
;
424 char *packfile_name
= NULL
, *p
, *dot
;
425 struct got_object_id id
;
431 packdir_path
= got_repo_get_path_objects_pack(repo
);
432 if (packdir_path
== NULL
)
433 return got_error_from_errno("got_repo_get_path_objects_pack");
435 if (!got_path_is_child(packfile_path
, packdir_path
,
436 strlen(packdir_path
))) {
437 err
= got_error_path(packfile_path
, GOT_ERR_BAD_PATH
);
442 err
= got_path_basename(&packfile_name
, packfile_path
);
447 if (strncmp(p
, "pack-", 5) != 0) {
448 err
= got_error_fmt(GOT_ERR_BAD_PATH
,
449 "'%s' is not a valid pack file name",
454 dot
= strchr(p
, '.');
456 err
= got_error_fmt(GOT_ERR_BAD_PATH
,
457 "'%s' is not a valid pack file name",
461 if (strcmp(dot
+ 1, "pack") != 0) {
462 err
= got_error_fmt(GOT_ERR_BAD_PATH
,
463 "'%s' is not a valid pack file name",
468 if (!got_parse_sha1_digest(id
.sha1
, p
)) {
469 err
= got_error_fmt(GOT_ERR_BAD_PATH
,
470 "'%s' is not a valid pack file name",
475 *pack_hash
= got_object_id_dup(&id
);
476 if (*pack_hash
== NULL
) {
477 err
= got_error_from_errno("got_object_id_dup");
481 packfd
= open(packfile_path
, O_RDONLY
| O_NOFOLLOW
);
483 err
= got_error_from_errno2("open", packfile_path
);
487 *packfile
= fdopen(packfd
, "r");
488 if (*packfile
== NULL
) {
489 err
= got_error_from_errno2("fdopen", packfile_path
);
494 if (packfd
!= -1 && close(packfd
) == -1 && err
== NULL
)
495 err
= got_error_from_errno2("close", packfile_path
);
504 const struct got_error
*
505 got_repo_list_pack(FILE *packfile
, struct got_object_id
*pack_hash
,
506 struct got_repository
*repo
, got_pack_list_cb list_cb
, void *list_arg
,
507 got_cancel_cb cancel_cb
, void *cancel_arg
)
509 const struct got_error
*err
= NULL
;
510 char *id_str
= NULL
, *idxpath
= NULL
, *packpath
= NULL
;
511 struct got_packidx
*packidx
= NULL
;
512 struct got_pack
*pack
= NULL
;
515 err
= got_object_id_str(&id_str
, pack_hash
);
519 if (asprintf(&packpath
, "%s/pack-%s.pack",
520 GOT_OBJECTS_PACK_DIR
, id_str
) == -1) {
521 err
= got_error_from_errno("asprintf");
524 if (asprintf(&idxpath
, "%s/pack-%s.idx",
525 GOT_OBJECTS_PACK_DIR
, id_str
) == -1) {
526 err
= got_error_from_errno("asprintf");
530 err
= got_packidx_open(&packidx
, got_repo_get_fd(repo
), idxpath
, 1);
534 err
= got_repo_cache_pack(&pack
, repo
, packpath
, packidx
);
538 nobj
= be32toh(packidx
->hdr
.fanout_table
[0xff]);
539 for (i
= 0; i
< nobj
; i
++) {
540 struct got_packidx_object_id
*oid
;
541 struct got_object_id id
, base_id
;
542 off_t offset
, base_offset
= 0;
548 err
= cancel_cb(cancel_arg
);
552 oid
= &packidx
->hdr
.sorted_ids
[i
];
553 memcpy(id
.sha1
, oid
->sha1
, SHA1_DIGEST_LENGTH
);
555 offset
= got_packidx_get_object_offset(packidx
, i
);
557 err
= got_error(GOT_ERR_BAD_PACKIDX
);
561 err
= got_pack_parse_object_type_and_size(&type
, &size
, &tslen
,
567 case GOT_OBJ_TYPE_OFFSET_DELTA
:
568 err
= got_pack_parse_offset_delta(&base_offset
, &len
,
569 pack
, offset
, tslen
);
573 case GOT_OBJ_TYPE_REF_DELTA
:
574 err
= got_pack_parse_ref_delta(&base_id
,
575 pack
, offset
, tslen
);
580 err
= (*list_cb
)(list_arg
, &id
, type
, offset
, size
,
581 base_offset
, &base_id
);
591 got_packidx_close(packidx
);
595 static const struct got_error
*
596 get_loose_object_ids(struct got_object_idset
**loose_ids
, off_t
*ondisk_size
,
597 got_cleanup_progress_cb progress_cb
, void *progress_arg
,
598 struct got_repository
*repo
)
600 const struct got_error
*err
= NULL
;
601 char *path_objects
= NULL
, *path
= NULL
;
603 struct got_object
*obj
= NULL
;
604 struct got_object_id id
;
609 *loose_ids
= got_object_idset_alloc();
610 if (*loose_ids
== NULL
)
611 return got_error_from_errno("got_object_idset_alloc");
613 path_objects
= got_repo_get_path_objects(repo
);
614 if (path_objects
== NULL
) {
615 err
= got_error_from_errno("got_repo_get_path_objects");
619 for (i
= 0; i
<= 0xff; i
++) {
622 if (asprintf(&path
, "%s/%.2x", path_objects
, i
) == -1) {
623 err
= got_error_from_errno("asprintf");
629 if (errno
== ENOENT
) {
633 err
= got_error_from_errno2("opendir", path
);
637 while ((dent
= readdir(dir
)) != NULL
) {
640 if (strcmp(dent
->d_name
, ".") == 0 ||
641 strcmp(dent
->d_name
, "..") == 0)
644 if (asprintf(&id_str
, "%.2x%s", i
, dent
->d_name
) == -1) {
645 err
= got_error_from_errno("asprintf");
649 memset(&id
, 0, sizeof(id
));
650 if (!got_parse_sha1_digest(id
.sha1
, id_str
)) {
656 err
= got_object_open_loose_fd(&fd
, &id
, repo
);
659 if (fstat(fd
, &sb
) == -1) {
660 err
= got_error_from_errno("fstat");
663 err
= got_object_read_header_privsep(&obj
, &id
, repo
,
667 fd
= -1; /* already closed */
670 case GOT_OBJ_TYPE_COMMIT
:
671 case GOT_OBJ_TYPE_TREE
:
672 case GOT_OBJ_TYPE_BLOB
:
673 case GOT_OBJ_TYPE_TAG
:
676 err
= got_error_fmt(GOT_ERR_OBJ_TYPE
,
680 got_object_close(obj
);
682 (*ondisk_size
) += sb
.st_size
;
683 err
= got_object_idset_add(*loose_ids
, &id
, NULL
);
687 err
= progress_cb(progress_arg
,
688 got_object_idset_num_elements(*loose_ids
),
695 if (closedir(dir
) != 0) {
696 err
= got_error_from_errno("closedir");
705 if (dir
&& closedir(dir
) != 0 && err
== NULL
)
706 err
= got_error_from_errno("closedir");
707 if (fd
!= -1 && close(fd
) == -1 && err
== NULL
)
708 err
= got_error_from_errno("close");
710 got_object_idset_free(*loose_ids
);
714 got_object_close(obj
);
720 static const struct got_error
*
721 preserve_loose_object(struct got_object_idset
*loose_ids
,
722 struct got_object_id
*id
, struct got_repository
*repo
, int *npacked
)
724 const struct got_error
*err
= NULL
;
725 struct got_object
*obj
;
727 if (!got_object_idset_contains(loose_ids
, id
))
731 * Try to open this object from a pack file. This ensures that
732 * we do in fact have a valid packed copy of the object. Otherwise
733 * we should not delete the loose representation of this object.
735 err
= got_object_open_packed(&obj
, id
, repo
);
737 got_object_close(obj
);
739 * The object is referenced and packed.
740 * We can purge the redundantly stored loose object.
744 } else if (err
->code
!= GOT_ERR_NO_OBJ
)
748 * This object is referenced and not packed.
749 * Remove it from our purge set.
751 return got_object_idset_remove(NULL
, loose_ids
, id
);
754 static const struct got_error
*
755 load_tree_entries(struct got_object_id_queue
*ids
,
756 struct got_object_idset
*loose_ids
,
757 struct got_object_idset
*traversed_ids
, struct got_object_id
*tree_id
,
758 const char *dpath
, struct got_repository
*repo
, int *npacked
,
759 got_cancel_cb cancel_cb
, void *cancel_arg
)
761 const struct got_error
*err
;
762 struct got_tree_object
*tree
;
766 err
= got_object_open_as_tree(&tree
, repo
, tree_id
);
770 for (i
= 0; i
< got_object_tree_get_nentries(tree
); i
++) {
771 struct got_tree_entry
*e
= got_object_tree_get_entry(tree
, i
);
772 struct got_object_id
*id
= got_tree_entry_get_id(e
);
773 mode_t mode
= got_tree_entry_get_mode(e
);
776 err
= (*cancel_cb
)(cancel_arg
);
781 if (got_object_tree_entry_is_symlink(e
) ||
782 got_object_tree_entry_is_submodule(e
) ||
783 got_object_idset_contains(traversed_ids
, id
))
786 if (asprintf(&p
, "%s%s%s", dpath
, dpath
[0] != '\0' ? "/" : "",
787 got_tree_entry_get_name(e
)) == -1) {
788 err
= got_error_from_errno("asprintf");
793 struct got_object_qid
*qid
;
794 err
= got_object_qid_alloc(&qid
, id
);
797 STAILQ_INSERT_TAIL(ids
, qid
, entry
);
798 } else if (S_ISREG(mode
)) {
799 /* This blob is referenced. */
800 err
= preserve_loose_object(loose_ids
, id
, repo
,
804 err
= got_object_idset_add(traversed_ids
, id
, NULL
);
813 got_object_tree_close(tree
);
818 static const struct got_error
*
819 load_tree(struct got_object_idset
*loose_ids
,
820 struct got_object_idset
*traversed_ids
, struct got_object_id
*tree_id
,
821 const char *dpath
, struct got_repository
*repo
, int *npacked
,
822 got_cancel_cb cancel_cb
, void *cancel_arg
)
824 const struct got_error
*err
= NULL
;
825 struct got_object_id_queue tree_ids
;
826 struct got_object_qid
*qid
;
828 err
= got_object_qid_alloc(&qid
, tree_id
);
832 STAILQ_INIT(&tree_ids
);
833 STAILQ_INSERT_TAIL(&tree_ids
, qid
, entry
);
835 while (!STAILQ_EMPTY(&tree_ids
)) {
837 err
= (*cancel_cb
)(cancel_arg
);
842 qid
= STAILQ_FIRST(&tree_ids
);
843 STAILQ_REMOVE_HEAD(&tree_ids
, entry
);
845 if (got_object_idset_contains(traversed_ids
, qid
->id
)) {
846 got_object_qid_free(qid
);
850 err
= got_object_idset_add(traversed_ids
, qid
->id
, NULL
);
852 got_object_qid_free(qid
);
856 /* This tree is referenced. */
857 err
= preserve_loose_object(loose_ids
, qid
->id
, repo
, npacked
);
861 err
= load_tree_entries(&tree_ids
, loose_ids
, traversed_ids
,
862 qid
->id
, dpath
, repo
, npacked
, cancel_cb
, cancel_arg
);
863 got_object_qid_free(qid
);
868 got_object_id_queue_free(&tree_ids
);
872 static const struct got_error
*
873 load_commit_or_tag(struct got_object_idset
*loose_ids
, int *ncommits
,
874 int *npacked
, struct got_object_idset
*traversed_ids
,
875 struct got_object_id
*id
, struct got_repository
*repo
,
876 got_cleanup_progress_cb progress_cb
, void *progress_arg
, int nloose
,
877 got_cancel_cb cancel_cb
, void *cancel_arg
)
879 const struct got_error
*err
;
880 struct got_commit_object
*commit
= NULL
;
881 struct got_tag_object
*tag
= NULL
;
882 struct got_object_id
*tree_id
= NULL
;
883 struct got_object_id_queue ids
;
884 struct got_object_qid
*qid
;
887 err
= got_object_qid_alloc(&qid
, id
);
892 STAILQ_INSERT_TAIL(&ids
, qid
, entry
);
894 while (!STAILQ_EMPTY(&ids
)) {
896 err
= (*cancel_cb
)(cancel_arg
);
901 qid
= STAILQ_FIRST(&ids
);
902 STAILQ_REMOVE_HEAD(&ids
, entry
);
904 if (got_object_idset_contains(traversed_ids
, qid
->id
)) {
905 got_object_qid_free(qid
);
910 err
= got_object_idset_add(traversed_ids
, qid
->id
, NULL
);
914 /* This commit or tag is referenced. */
915 err
= preserve_loose_object(loose_ids
, qid
->id
, repo
, npacked
);
919 err
= got_object_get_type(&obj_type
, repo
, qid
->id
);
923 case GOT_OBJ_TYPE_COMMIT
:
924 err
= got_object_open_as_commit(&commit
, repo
, qid
->id
);
928 case GOT_OBJ_TYPE_TAG
:
929 err
= got_object_open_as_tag(&tag
, repo
, qid
->id
);
934 /* should not happen */
935 err
= got_error(GOT_ERR_OBJ_TYPE
);
939 /* Find a tree object to scan. */
941 tree_id
= got_object_commit_get_tree_id(commit
);
943 obj_type
= got_object_tag_get_object_type(tag
);
945 case GOT_OBJ_TYPE_COMMIT
:
946 err
= got_object_open_as_commit(&commit
, repo
,
947 got_object_tag_get_object_id(tag
));
950 tree_id
= got_object_commit_get_tree_id(commit
);
952 case GOT_OBJ_TYPE_TREE
:
953 tree_id
= got_object_tag_get_object_id(tag
);
957 * Tag points at something other than a
958 * commit or tree. Leave this weird tag object
959 * and the object it points to on disk.
961 err
= got_object_idset_remove(NULL
, loose_ids
,
963 if (err
&& err
->code
!= GOT_ERR_NO_OBJ
)
965 err
= got_object_idset_remove(NULL
, loose_ids
,
966 got_object_tag_get_object_id(tag
));
967 if (err
&& err
->code
!= GOT_ERR_NO_OBJ
)
975 err
= load_tree(loose_ids
, traversed_ids
, tree_id
, "",
976 repo
, npacked
, cancel_cb
, cancel_arg
);
982 (*ncommits
)++; /* scanned tags are counted as commits */
985 err
= progress_cb(progress_arg
, nloose
, *ncommits
, -1);
991 /* Find parent commits to scan. */
992 const struct got_object_id_queue
*parent_ids
;
993 parent_ids
= got_object_commit_get_parent_ids(commit
);
994 err
= got_object_id_queue_copy(parent_ids
, &ids
);
997 got_object_commit_close(commit
);
1001 got_object_tag_close(tag
);
1004 got_object_qid_free(qid
);
1009 got_object_qid_free(qid
);
1011 got_object_commit_close(commit
);
1013 got_object_tag_close(tag
);
1014 got_object_id_queue_free(&ids
);
1018 struct purge_loose_object_arg
{
1019 struct got_repository
*repo
;
1020 got_cleanup_progress_cb progress_cb
;
1031 static const struct got_error
*
1032 purge_loose_object(struct got_object_id
*id
, void *data
, void *arg
)
1034 struct purge_loose_object_arg
*a
= arg
;
1035 const struct got_error
*err
, *unlock_err
= NULL
;
1039 struct got_lockfile
*lf
= NULL
;
1041 err
= got_object_get_path(&path
, id
, a
->repo
);
1045 err
= got_object_open_loose_fd(&fd
, id
, a
->repo
);
1049 if (fstat(fd
, &sb
) == -1) {
1050 err
= got_error_from_errno("fstat");
1055 * Do not delete objects which are younger than our maximum
1056 * modification time threshold. This prevents a race where
1057 * new objects which are being added to the repository
1058 * concurrently would be deleted.
1060 if (a
->ignore_mtime
|| sb
.st_mtime
<= a
->max_mtime
) {
1062 err
= got_lockfile_lock(&lf
, path
, -1);
1065 if (unlink(path
) == -1) {
1066 err
= got_error_from_errno2("unlink", path
);
1072 a
->size_purged
+= sb
.st_size
;
1073 if (a
->progress_cb
) {
1074 err
= a
->progress_cb(a
->progress_arg
, a
->nloose
,
1075 a
->ncommits
, a
->npurged
);
1079 if (fd
!= -1 && close(fd
) == -1 && err
== NULL
)
1080 err
= got_error_from_errno("close");
1083 unlock_err
= got_lockfile_unlock(lf
, -1);
1084 return err
? err
: unlock_err
;
1087 const struct got_error
*
1088 got_repo_purge_unreferenced_loose_objects(struct got_repository
*repo
,
1089 off_t
*size_before
, off_t
*size_after
, int *npacked
, int dry_run
,
1090 int ignore_mtime
, got_cleanup_progress_cb progress_cb
, void *progress_arg
,
1091 got_cancel_cb cancel_cb
, void *cancel_arg
)
1093 const struct got_error
*err
;
1094 struct got_object_idset
*loose_ids
;
1095 struct got_object_idset
*traversed_ids
;
1096 struct got_object_id
**referenced_ids
;
1097 int i
, nreferenced
, nloose
, ncommits
= 0;
1098 struct got_reflist_head refs
;
1099 struct got_reflist_entry
*re
;
1100 struct purge_loose_object_arg arg
;
1101 time_t max_mtime
= 0;
1109 err
= get_loose_object_ids(&loose_ids
, size_before
,
1110 progress_cb
, progress_arg
, repo
);
1113 nloose
= got_object_idset_num_elements(loose_ids
);
1115 got_object_idset_free(loose_ids
);
1119 traversed_ids
= got_object_idset_alloc();
1120 if (traversed_ids
== NULL
) {
1121 err
= got_error_from_errno("got_object_idset_alloc");
1125 err
= got_ref_list(&refs
, repo
, "", got_ref_cmp_by_name
, NULL
);
1128 if (!ignore_mtime
) {
1129 TAILQ_FOREACH(re
, &refs
, entry
) {
1130 time_t mtime
= got_ref_get_mtime(re
->ref
);
1131 if (mtime
> max_mtime
)
1135 * For safety, keep objects created within 10 minutes
1136 * before the youngest reference was created.
1138 if (max_mtime
>= 600)
1142 err
= get_reflist_object_ids(&referenced_ids
, &nreferenced
,
1143 (1 << GOT_OBJ_TYPE_COMMIT
) | (1 << GOT_OBJ_TYPE_TAG
),
1144 &refs
, repo
, cancel_cb
, cancel_arg
);
1148 for (i
= 0; i
< nreferenced
; i
++) {
1149 struct got_object_id
*id
= referenced_ids
[i
];
1150 err
= load_commit_or_tag(loose_ids
, &ncommits
, npacked
,
1151 traversed_ids
, id
, repo
, progress_cb
, progress_arg
, nloose
,
1152 cancel_cb
, cancel_arg
);
1157 /* Produce a final progress report in case no objects can be purged. */
1158 if (got_object_idset_num_elements(loose_ids
) == 0 && progress_cb
) {
1159 err
= progress_cb(progress_arg
, nloose
, ncommits
, 0);
1164 /* Any remaining loose objects are unreferenced and can be purged. */
1166 arg
.progress_arg
= progress_arg
;
1167 arg
.progress_cb
= progress_cb
;
1168 arg
.nloose
= nloose
;
1170 arg
.size_purged
= 0;
1171 arg
.ncommits
= ncommits
;
1172 arg
.dry_run
= dry_run
;
1173 arg
.max_mtime
= max_mtime
;
1174 arg
.ignore_mtime
= ignore_mtime
;
1175 err
= got_object_idset_for_each(loose_ids
, purge_loose_object
, &arg
);
1178 *size_after
= *size_before
- arg
.size_purged
;
1180 got_object_idset_free(loose_ids
);
1181 got_object_idset_free(traversed_ids
);
1185 static const struct got_error
*
1186 remove_packidx(int dir_fd
, const char *relpath
)
1188 const struct got_error
*err
, *unlock_err
;
1189 struct got_lockfile
*lf
;
1191 err
= got_lockfile_lock(&lf
, relpath
, dir_fd
);
1194 if (unlinkat(dir_fd
, relpath
, 0) == -1)
1195 err
= got_error_from_errno("unlinkat");
1196 unlock_err
= got_lockfile_unlock(lf
, dir_fd
);
1197 return err
? err
: unlock_err
;
1200 const struct got_error
*
1201 got_repo_remove_lonely_packidx(struct got_repository
*repo
, int dry_run
,
1202 got_lonely_packidx_progress_cb progress_cb
, void *progress_arg
,
1203 got_cancel_cb cancel_cb
, void *cancel_arg
)
1205 const struct got_error
*err
;
1206 DIR *packdir
= NULL
;
1207 struct dirent
*dent
;
1208 char *pack_relpath
= NULL
;
1212 packdir_fd
= openat(got_repo_get_fd(repo
),
1213 GOT_OBJECTS_PACK_DIR
, O_DIRECTORY
);
1214 if (packdir_fd
== -1) {
1215 if (errno
== ENOENT
)
1217 return got_error_from_errno_fmt("openat: %s/%s",
1218 got_repo_get_path_git_dir(repo
),
1219 GOT_OBJECTS_PACK_DIR
);
1222 packdir
= fdopendir(packdir_fd
);
1223 if (packdir
== NULL
) {
1224 err
= got_error_from_errno("fdopendir");
1228 while ((dent
= readdir(packdir
)) != NULL
) {
1230 err
= cancel_cb(cancel_arg
);
1235 if (!got_repo_is_packidx_filename(dent
->d_name
,
1236 strlen(dent
->d_name
)))
1239 err
= got_packidx_get_packfile_path(&pack_relpath
,
1244 if (fstatat(packdir_fd
, pack_relpath
, &sb
, 0) != -1) {
1246 pack_relpath
= NULL
;
1249 if (errno
!= ENOENT
) {
1250 err
= got_error_from_errno_fmt("fstatat: %s/%s/%s",
1251 got_repo_get_path_git_dir(repo
),
1252 GOT_OBJECTS_PACK_DIR
,
1258 err
= remove_packidx(packdir_fd
, dent
->d_name
);
1264 if (asprintf(&path
, "%s/%s/%s",
1265 got_repo_get_path_git_dir(repo
),
1266 GOT_OBJECTS_PACK_DIR
,
1267 dent
->d_name
) == -1) {
1268 err
= got_error_from_errno("asprintf");
1271 err
= progress_cb(progress_arg
, path
);
1277 pack_relpath
= NULL
;
1280 if (packdir
&& closedir(packdir
) != 0 && err
== NULL
)
1281 err
= got_error_from_errno("closedir");