2 * Copyright (c) 2020 Stefan Sperling <stsp@openbsd.org>
4 * Permission to use, copy, modify, and distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 #include <sys/types.h>
18 #include <sys/queue.h>
22 #include <sys/socket.h>
35 #include "got_error.h"
36 #include "got_cancel.h"
37 #include "got_object.h"
38 #include "got_reference.h"
39 #include "got_repository.h"
40 #include "got_repository_admin.h"
41 #include "got_opentemp.h"
44 #include "got_lib_delta.h"
45 #include "got_lib_object.h"
46 #include "got_lib_object_idset.h"
47 #include "got_lib_object_cache.h"
48 #include "got_lib_pack.h"
49 #include "got_lib_privsep.h"
50 #include "got_lib_repository.h"
51 #include "got_lib_pack_create.h"
52 #include "got_lib_sha1.h"
53 #include "got_lib_lockfile.h"
56 #define nitems(_a) (sizeof((_a)) / sizeof((_a)[0]))
59 static const struct got_error
*
60 get_reflist_object_ids(struct got_object_id
***ids
, int *nobjects
,
61 unsigned int wanted_obj_type_mask
, struct got_reflist_head
*refs
,
62 struct got_repository
*repo
,
63 got_cancel_cb cancel_cb
, void *cancel_arg
)
65 const struct got_error
*err
= NULL
;
66 const size_t alloc_chunksz
= 256;
68 struct got_reflist_entry
*re
;
74 *ids
= reallocarray(NULL
, alloc_chunksz
, sizeof(struct got_object_id
*));
76 return got_error_from_errno("reallocarray");
77 nalloc
= alloc_chunksz
;
79 TAILQ_FOREACH(re
, refs
, entry
) {
80 struct got_object_id
*id
;
83 err
= cancel_cb(cancel_arg
);
88 err
= got_ref_resolve(&id
, repo
, re
->ref
);
92 if (wanted_obj_type_mask
!= GOT_OBJ_TYPE_ANY
) {
94 err
= got_object_get_type(&obj_type
, repo
, id
);
97 if ((wanted_obj_type_mask
& (1 << obj_type
)) == 0) {
104 if (nalloc
<= *nobjects
) {
105 struct got_object_id
**new;
106 new = recallocarray(*ids
, nalloc
,
107 nalloc
+ alloc_chunksz
,
108 sizeof(struct got_object_id
*));
110 err
= got_error_from_errno(
115 nalloc
+= alloc_chunksz
;
117 (*ids
)[*nobjects
] = id
;
118 if ((*ids
)[*nobjects
] == NULL
) {
119 err
= got_error_from_errno("got_object_id_dup");
126 for (i
= 0; i
< *nobjects
; i
++)
135 const struct got_error
*
136 got_repo_pack_objects(FILE **packfile
, struct got_object_id
**pack_hash
,
137 struct got_reflist_head
*include_refs
,
138 struct got_reflist_head
*exclude_refs
, struct got_repository
*repo
,
139 int loose_obj_only
, got_pack_progress_cb progress_cb
, void *progress_arg
,
140 got_cancel_cb cancel_cb
, void *cancel_arg
)
142 const struct got_error
*err
= NULL
;
143 struct got_object_id
**ours
= NULL
, **theirs
= NULL
;
144 int nours
= 0, ntheirs
= 0, packfd
= -1, i
;
145 char *tmpfile_path
= NULL
, *path
= NULL
, *packfile_path
= NULL
;
146 char *sha1_str
= NULL
;
151 if (asprintf(&path
, "%s/%s/packing.pack",
152 got_repo_get_path_git_dir(repo
), GOT_OBJECTS_PACK_DIR
) == -1) {
153 err
= got_error_from_errno("asprintf");
156 err
= got_opentemp_named_fd(&tmpfile_path
, &packfd
, path
);
160 if (fchmod(packfd
, GOT_DEFAULT_FILE_MODE
) != 0) {
161 err
= got_error_from_errno2("fchmod", tmpfile_path
);
165 *packfile
= fdopen(packfd
, "w");
166 if (*packfile
== NULL
) {
167 err
= got_error_from_errno2("fdopen", tmpfile_path
);
172 err
= get_reflist_object_ids(&ours
, &nours
,
173 (1 << GOT_OBJ_TYPE_COMMIT
) | (1 << GOT_OBJ_TYPE_TAG
),
174 include_refs
, repo
, cancel_cb
, cancel_arg
);
179 err
= got_error(GOT_ERR_CANNOT_PACK
);
183 if (!TAILQ_EMPTY(exclude_refs
)) {
184 err
= get_reflist_object_ids(&theirs
, &ntheirs
,
185 (1 << GOT_OBJ_TYPE_COMMIT
) | (1 << GOT_OBJ_TYPE_TAG
),
187 cancel_cb
, cancel_arg
);
192 *pack_hash
= calloc(1, sizeof(**pack_hash
));
193 if (*pack_hash
== NULL
) {
194 err
= got_error_from_errno("calloc");
198 err
= got_pack_create((*pack_hash
)->sha1
, *packfile
, theirs
, ntheirs
,
199 ours
, nours
, repo
, loose_obj_only
, 0, progress_cb
, progress_arg
,
200 cancel_cb
, cancel_arg
);
204 err
= got_object_id_str(&sha1_str
, *pack_hash
);
207 if (asprintf(&packfile_path
, "%s/%s/pack-%s.pack",
208 got_repo_get_path_git_dir(repo
), GOT_OBJECTS_PACK_DIR
,
210 err
= got_error_from_errno("asprintf");
214 if (fflush(*packfile
) == -1) {
215 err
= got_error_from_errno("fflush");
218 if (fseek(*packfile
, 0L, SEEK_SET
) == -1) {
219 err
= got_error_from_errno("fseek");
222 if (rename(tmpfile_path
, packfile_path
) == -1) {
223 err
= got_error_from_errno3("rename", tmpfile_path
,
230 for (i
= 0; i
< nours
; i
++)
233 for (i
= 0; i
< ntheirs
; i
++)
236 if (packfd
!= -1 && close(packfd
) == -1 && err
== NULL
)
237 err
= got_error_from_errno2("close", packfile_path
);
238 if (tmpfile_path
&& unlink(tmpfile_path
) == -1 && err
== NULL
)
239 err
= got_error_from_errno2("unlink", tmpfile_path
);
254 const struct got_error
*
255 got_repo_index_pack(FILE *packfile
, struct got_object_id
*pack_hash
,
256 struct got_repository
*repo
,
257 got_pack_index_progress_cb progress_cb
, void *progress_arg
,
258 got_cancel_cb cancel_cb
, void *cancel_arg
)
263 int npackfd
= -1, idxfd
= -1, nidxfd
= -1;
265 int idxstatus
, done
= 0;
266 const struct got_error
*err
;
267 struct imsgbuf idxibuf
;
269 char *tmpidxpath
= NULL
;
270 char *packfile_path
= NULL
, *idxpath
= NULL
, *id_str
= NULL
;
271 const char *repo_path
= got_repo_get_path_git_dir(repo
);
274 for (i
= 0; i
< nitems(tmpfds
); i
++)
277 if (asprintf(&path
, "%s/%s/indexing.idx",
278 repo_path
, GOT_OBJECTS_PACK_DIR
) == -1) {
279 err
= got_error_from_errno("asprintf");
282 err
= got_opentemp_named_fd(&tmpidxpath
, &idxfd
, path
);
286 if (fchmod(idxfd
, GOT_DEFAULT_FILE_MODE
) != 0) {
287 err
= got_error_from_errno2("fchmod", tmpidxpath
);
293 err
= got_error_from_errno("dup");
297 for (i
= 0; i
< nitems(tmpfds
); i
++) {
298 tmpfds
[i
] = got_opentempfd();
299 if (tmpfds
[i
] == -1) {
300 err
= got_error_from_errno("got_opentempfd");
305 err
= got_object_id_str(&id_str
, pack_hash
);
309 if (asprintf(&packfile_path
, "%s/%s/pack-%s.pack",
310 repo_path
, GOT_OBJECTS_PACK_DIR
, id_str
) == -1) {
311 err
= got_error_from_errno("asprintf");
315 if (fstat(fileno(packfile
), &sb
) == -1) {
316 err
= got_error_from_errno2("fstat", packfile_path
);
320 if (asprintf(&idxpath
, "%s/%s/pack-%s.idx",
321 repo_path
, GOT_OBJECTS_PACK_DIR
, id_str
) == -1) {
322 err
= got_error_from_errno("asprintf");
326 if (socketpair(AF_UNIX
, SOCK_STREAM
, PF_UNSPEC
, imsg_idxfds
) == -1) {
327 err
= got_error_from_errno("socketpair");
332 err
= got_error_from_errno("fork");
334 } else if (idxpid
== 0)
335 got_privsep_exec_child(imsg_idxfds
,
336 GOT_PATH_PROG_INDEX_PACK
, packfile_path
);
337 if (close(imsg_idxfds
[1]) == -1) {
338 err
= got_error_from_errno("close");
341 imsg_init(&idxibuf
, imsg_idxfds
[0]);
343 npackfd
= dup(fileno(packfile
));
345 err
= got_error_from_errno("dup");
348 err
= got_privsep_send_index_pack_req(&idxibuf
, pack_hash
->sha1
,
353 err
= got_privsep_send_index_pack_outfd(&idxibuf
, nidxfd
);
357 for (i
= 0; i
< nitems(tmpfds
); i
++) {
358 err
= got_privsep_send_tmpfd(&idxibuf
, tmpfds
[i
]);
365 int nobj_total
, nobj_indexed
, nobj_loose
, nobj_resolved
;
368 err
= cancel_cb(cancel_arg
);
373 err
= got_privsep_recv_index_progress(&done
, &nobj_total
,
374 &nobj_indexed
, &nobj_loose
, &nobj_resolved
,
378 if (nobj_indexed
!= 0) {
379 err
= progress_cb(progress_arg
, sb
.st_size
,
380 nobj_total
, nobj_indexed
, nobj_loose
,
385 imsg_clear(&idxibuf
);
387 if (close(imsg_idxfds
[0]) == -1) {
388 err
= got_error_from_errno("close");
391 if (waitpid(idxpid
, &idxstatus
, 0) == -1) {
392 err
= got_error_from_errno("waitpid");
396 if (rename(tmpidxpath
, idxpath
) == -1) {
397 err
= got_error_from_errno3("rename", tmpidxpath
, idxpath
);
404 if (tmpidxpath
&& unlink(tmpidxpath
) == -1 && err
== NULL
)
405 err
= got_error_from_errno2("unlink", tmpidxpath
);
406 if (npackfd
!= -1 && close(npackfd
) == -1 && err
== NULL
)
407 err
= got_error_from_errno("close");
408 if (idxfd
!= -1 && close(idxfd
) == -1 && err
== NULL
)
409 err
= got_error_from_errno("close");
410 for (i
= 0; i
< nitems(tmpfds
); i
++) {
411 if (tmpfds
[i
] != -1 && close(tmpfds
[i
]) == -1 && err
== NULL
)
412 err
= got_error_from_errno("close");
420 const struct got_error
*
421 got_repo_find_pack(FILE **packfile
, struct got_object_id
**pack_hash
,
422 struct got_repository
*repo
, const char *packfile_path
)
424 const struct got_error
*err
= NULL
;
425 const char *packdir_path
= NULL
;
426 char *packfile_name
= NULL
, *p
, *dot
;
427 struct got_object_id id
;
433 packdir_path
= got_repo_get_path_objects_pack(repo
);
434 if (packdir_path
== NULL
)
435 return got_error_from_errno("got_repo_get_path_objects_pack");
437 if (!got_path_is_child(packfile_path
, packdir_path
,
438 strlen(packdir_path
))) {
439 err
= got_error_path(packfile_path
, GOT_ERR_BAD_PATH
);
444 err
= got_path_basename(&packfile_name
, packfile_path
);
449 if (strncmp(p
, "pack-", 5) != 0) {
450 err
= got_error_fmt(GOT_ERR_BAD_PATH
,
451 "'%s' is not a valid pack file name",
456 dot
= strchr(p
, '.');
458 err
= got_error_fmt(GOT_ERR_BAD_PATH
,
459 "'%s' is not a valid pack file name",
463 if (strcmp(dot
+ 1, "pack") != 0) {
464 err
= got_error_fmt(GOT_ERR_BAD_PATH
,
465 "'%s' is not a valid pack file name",
470 if (!got_parse_sha1_digest(id
.sha1
, p
)) {
471 err
= got_error_fmt(GOT_ERR_BAD_PATH
,
472 "'%s' is not a valid pack file name",
477 *pack_hash
= got_object_id_dup(&id
);
478 if (*pack_hash
== NULL
) {
479 err
= got_error_from_errno("got_object_id_dup");
483 packfd
= open(packfile_path
, O_RDONLY
| O_NOFOLLOW
);
485 err
= got_error_from_errno2("open", packfile_path
);
489 *packfile
= fdopen(packfd
, "r");
490 if (*packfile
== NULL
) {
491 err
= got_error_from_errno2("fdopen", packfile_path
);
496 if (packfd
!= -1 && close(packfd
) == -1 && err
== NULL
)
497 err
= got_error_from_errno2("close", packfile_path
);
506 const struct got_error
*
507 got_repo_list_pack(FILE *packfile
, struct got_object_id
*pack_hash
,
508 struct got_repository
*repo
, got_pack_list_cb list_cb
, void *list_arg
,
509 got_cancel_cb cancel_cb
, void *cancel_arg
)
511 const struct got_error
*err
= NULL
;
512 char *id_str
= NULL
, *idxpath
= NULL
, *packpath
= NULL
;
513 struct got_packidx
*packidx
= NULL
;
514 struct got_pack
*pack
= NULL
;
517 err
= got_object_id_str(&id_str
, pack_hash
);
521 if (asprintf(&packpath
, "%s/pack-%s.pack",
522 GOT_OBJECTS_PACK_DIR
, id_str
) == -1) {
523 err
= got_error_from_errno("asprintf");
526 if (asprintf(&idxpath
, "%s/pack-%s.idx",
527 GOT_OBJECTS_PACK_DIR
, id_str
) == -1) {
528 err
= got_error_from_errno("asprintf");
532 err
= got_packidx_open(&packidx
, got_repo_get_fd(repo
), idxpath
, 1);
536 err
= got_repo_cache_pack(&pack
, repo
, packpath
, packidx
);
540 nobj
= be32toh(packidx
->hdr
.fanout_table
[0xff]);
541 for (i
= 0; i
< nobj
; i
++) {
542 struct got_packidx_object_id
*oid
;
543 struct got_object_id id
, base_id
;
544 off_t offset
, base_offset
= 0;
550 err
= cancel_cb(cancel_arg
);
554 oid
= &packidx
->hdr
.sorted_ids
[i
];
555 memcpy(id
.sha1
, oid
->sha1
, SHA1_DIGEST_LENGTH
);
557 offset
= got_packidx_get_object_offset(packidx
, i
);
559 err
= got_error(GOT_ERR_BAD_PACKIDX
);
563 err
= got_pack_parse_object_type_and_size(&type
, &size
, &tslen
,
569 case GOT_OBJ_TYPE_OFFSET_DELTA
:
570 err
= got_pack_parse_offset_delta(&base_offset
, &len
,
571 pack
, offset
, tslen
);
575 case GOT_OBJ_TYPE_REF_DELTA
:
576 err
= got_pack_parse_ref_delta(&base_id
,
577 pack
, offset
, tslen
);
582 err
= (*list_cb
)(list_arg
, &id
, type
, offset
, size
,
583 base_offset
, &base_id
);
593 got_packidx_close(packidx
);
597 static const struct got_error
*
598 get_loose_object_ids(struct got_object_idset
**loose_ids
, off_t
*ondisk_size
,
599 got_cleanup_progress_cb progress_cb
, void *progress_arg
,
600 struct got_repository
*repo
)
602 const struct got_error
*err
= NULL
;
603 char *path_objects
= NULL
, *path
= NULL
;
605 struct got_object
*obj
= NULL
;
606 struct got_object_id id
;
611 *loose_ids
= got_object_idset_alloc();
612 if (*loose_ids
== NULL
)
613 return got_error_from_errno("got_object_idset_alloc");
615 path_objects
= got_repo_get_path_objects(repo
);
616 if (path_objects
== NULL
) {
617 err
= got_error_from_errno("got_repo_get_path_objects");
621 for (i
= 0; i
<= 0xff; i
++) {
624 if (asprintf(&path
, "%s/%.2x", path_objects
, i
) == -1) {
625 err
= got_error_from_errno("asprintf");
631 if (errno
== ENOENT
) {
635 err
= got_error_from_errno2("opendir", path
);
639 while ((dent
= readdir(dir
)) != NULL
) {
642 if (strcmp(dent
->d_name
, ".") == 0 ||
643 strcmp(dent
->d_name
, "..") == 0)
646 if (asprintf(&id_str
, "%.2x%s", i
, dent
->d_name
) == -1) {
647 err
= got_error_from_errno("asprintf");
651 memset(&id
, 0, sizeof(id
));
652 if (!got_parse_sha1_digest(id
.sha1
, id_str
)) {
658 err
= got_object_open_loose_fd(&fd
, &id
, repo
);
661 if (fstat(fd
, &sb
) == -1) {
662 err
= got_error_from_errno("fstat");
665 err
= got_object_read_header_privsep(&obj
, &id
, repo
,
669 fd
= -1; /* already closed */
672 case GOT_OBJ_TYPE_COMMIT
:
673 case GOT_OBJ_TYPE_TREE
:
674 case GOT_OBJ_TYPE_BLOB
:
675 case GOT_OBJ_TYPE_TAG
:
678 err
= got_error_fmt(GOT_ERR_OBJ_TYPE
,
682 got_object_close(obj
);
684 (*ondisk_size
) += sb
.st_size
;
685 err
= got_object_idset_add(*loose_ids
, &id
, NULL
);
689 err
= progress_cb(progress_arg
,
690 got_object_idset_num_elements(*loose_ids
),
697 if (closedir(dir
) != 0) {
698 err
= got_error_from_errno("closedir");
707 if (dir
&& closedir(dir
) != 0 && err
== NULL
)
708 err
= got_error_from_errno("closedir");
709 if (fd
!= -1 && close(fd
) == -1 && err
== NULL
)
710 err
= got_error_from_errno("close");
712 got_object_idset_free(*loose_ids
);
716 got_object_close(obj
);
722 static const struct got_error
*
723 preserve_loose_object(struct got_object_idset
*loose_ids
,
724 struct got_object_id
*id
, struct got_repository
*repo
, int *npacked
)
726 const struct got_error
*err
= NULL
;
727 struct got_object
*obj
;
729 if (!got_object_idset_contains(loose_ids
, id
))
733 * Try to open this object from a pack file. This ensures that
734 * we do in fact have a valid packed copy of the object. Otherwise
735 * we should not delete the loose representation of this object.
737 err
= got_object_open_packed(&obj
, id
, repo
);
739 got_object_close(obj
);
741 * The object is referenced and packed.
742 * We can purge the redundantly stored loose object.
746 } else if (err
->code
!= GOT_ERR_NO_OBJ
)
750 * This object is referenced and not packed.
751 * Remove it from our purge set.
753 return got_object_idset_remove(NULL
, loose_ids
, id
);
756 static const struct got_error
*
757 load_tree_entries(struct got_object_id_queue
*ids
,
758 struct got_object_idset
*loose_ids
,
759 struct got_object_idset
*traversed_ids
, struct got_object_id
*tree_id
,
760 const char *dpath
, struct got_repository
*repo
, int *npacked
,
761 got_cancel_cb cancel_cb
, void *cancel_arg
)
763 const struct got_error
*err
;
764 struct got_tree_object
*tree
;
768 err
= got_object_open_as_tree(&tree
, repo
, tree_id
);
772 for (i
= 0; i
< got_object_tree_get_nentries(tree
); i
++) {
773 struct got_tree_entry
*e
= got_object_tree_get_entry(tree
, i
);
774 struct got_object_id
*id
= got_tree_entry_get_id(e
);
775 mode_t mode
= got_tree_entry_get_mode(e
);
778 err
= (*cancel_cb
)(cancel_arg
);
783 if (got_object_tree_entry_is_symlink(e
) ||
784 got_object_tree_entry_is_submodule(e
) ||
785 got_object_idset_contains(traversed_ids
, id
))
788 if (asprintf(&p
, "%s%s%s", dpath
, dpath
[0] != '\0' ? "/" : "",
789 got_tree_entry_get_name(e
)) == -1) {
790 err
= got_error_from_errno("asprintf");
795 struct got_object_qid
*qid
;
796 err
= got_object_qid_alloc(&qid
, id
);
799 STAILQ_INSERT_TAIL(ids
, qid
, entry
);
800 } else if (S_ISREG(mode
)) {
801 /* This blob is referenced. */
802 err
= preserve_loose_object(loose_ids
, id
, repo
,
806 err
= got_object_idset_add(traversed_ids
, id
, NULL
);
815 got_object_tree_close(tree
);
820 static const struct got_error
*
821 load_tree(struct got_object_idset
*loose_ids
,
822 struct got_object_idset
*traversed_ids
, struct got_object_id
*tree_id
,
823 const char *dpath
, struct got_repository
*repo
, int *npacked
,
824 got_cancel_cb cancel_cb
, void *cancel_arg
)
826 const struct got_error
*err
= NULL
;
827 struct got_object_id_queue tree_ids
;
828 struct got_object_qid
*qid
;
830 err
= got_object_qid_alloc(&qid
, tree_id
);
834 STAILQ_INIT(&tree_ids
);
835 STAILQ_INSERT_TAIL(&tree_ids
, qid
, entry
);
837 while (!STAILQ_EMPTY(&tree_ids
)) {
839 err
= (*cancel_cb
)(cancel_arg
);
844 qid
= STAILQ_FIRST(&tree_ids
);
845 STAILQ_REMOVE_HEAD(&tree_ids
, entry
);
847 if (got_object_idset_contains(traversed_ids
, qid
->id
)) {
848 got_object_qid_free(qid
);
852 err
= got_object_idset_add(traversed_ids
, qid
->id
, NULL
);
854 got_object_qid_free(qid
);
858 /* This tree is referenced. */
859 err
= preserve_loose_object(loose_ids
, qid
->id
, repo
, npacked
);
863 err
= load_tree_entries(&tree_ids
, loose_ids
, traversed_ids
,
864 qid
->id
, dpath
, repo
, npacked
, cancel_cb
, cancel_arg
);
865 got_object_qid_free(qid
);
870 got_object_id_queue_free(&tree_ids
);
874 static const struct got_error
*
875 load_commit_or_tag(struct got_object_idset
*loose_ids
, int *ncommits
,
876 int *npacked
, struct got_object_idset
*traversed_ids
,
877 struct got_object_id
*id
, struct got_repository
*repo
,
878 got_cleanup_progress_cb progress_cb
, void *progress_arg
, int nloose
,
879 got_cancel_cb cancel_cb
, void *cancel_arg
)
881 const struct got_error
*err
;
882 struct got_commit_object
*commit
= NULL
;
883 struct got_tag_object
*tag
= NULL
;
884 struct got_object_id
*tree_id
= NULL
;
885 struct got_object_id_queue ids
;
886 struct got_object_qid
*qid
;
889 err
= got_object_qid_alloc(&qid
, id
);
894 STAILQ_INSERT_TAIL(&ids
, qid
, entry
);
896 while (!STAILQ_EMPTY(&ids
)) {
898 err
= (*cancel_cb
)(cancel_arg
);
903 qid
= STAILQ_FIRST(&ids
);
904 STAILQ_REMOVE_HEAD(&ids
, entry
);
906 if (got_object_idset_contains(traversed_ids
, qid
->id
)) {
907 got_object_qid_free(qid
);
912 err
= got_object_idset_add(traversed_ids
, qid
->id
, NULL
);
916 /* This commit or tag is referenced. */
917 err
= preserve_loose_object(loose_ids
, qid
->id
, repo
, npacked
);
921 err
= got_object_get_type(&obj_type
, repo
, qid
->id
);
925 case GOT_OBJ_TYPE_COMMIT
:
926 err
= got_object_open_as_commit(&commit
, repo
, qid
->id
);
930 case GOT_OBJ_TYPE_TAG
:
931 err
= got_object_open_as_tag(&tag
, repo
, qid
->id
);
936 /* should not happen */
937 err
= got_error(GOT_ERR_OBJ_TYPE
);
941 /* Find a tree object to scan. */
943 tree_id
= got_object_commit_get_tree_id(commit
);
945 obj_type
= got_object_tag_get_object_type(tag
);
947 case GOT_OBJ_TYPE_COMMIT
:
948 err
= got_object_open_as_commit(&commit
, repo
,
949 got_object_tag_get_object_id(tag
));
952 tree_id
= got_object_commit_get_tree_id(commit
);
954 case GOT_OBJ_TYPE_TREE
:
955 tree_id
= got_object_tag_get_object_id(tag
);
959 * Tag points at something other than a
960 * commit or tree. Leave this weird tag object
961 * and the object it points to on disk.
963 err
= got_object_idset_remove(NULL
, loose_ids
,
965 if (err
&& err
->code
!= GOT_ERR_NO_OBJ
)
967 err
= got_object_idset_remove(NULL
, loose_ids
,
968 got_object_tag_get_object_id(tag
));
969 if (err
&& err
->code
!= GOT_ERR_NO_OBJ
)
977 err
= load_tree(loose_ids
, traversed_ids
, tree_id
, "",
978 repo
, npacked
, cancel_cb
, cancel_arg
);
984 (*ncommits
)++; /* scanned tags are counted as commits */
987 err
= progress_cb(progress_arg
, nloose
, *ncommits
, -1);
993 /* Find parent commits to scan. */
994 const struct got_object_id_queue
*parent_ids
;
995 parent_ids
= got_object_commit_get_parent_ids(commit
);
996 err
= got_object_id_queue_copy(parent_ids
, &ids
);
999 got_object_commit_close(commit
);
1003 got_object_tag_close(tag
);
1006 got_object_qid_free(qid
);
1011 got_object_qid_free(qid
);
1013 got_object_commit_close(commit
);
1015 got_object_tag_close(tag
);
1016 got_object_id_queue_free(&ids
);
1020 struct purge_loose_object_arg
{
1021 struct got_repository
*repo
;
1022 got_cleanup_progress_cb progress_cb
;
1033 static const struct got_error
*
1034 purge_loose_object(struct got_object_id
*id
, void *data
, void *arg
)
1036 struct purge_loose_object_arg
*a
= arg
;
1037 const struct got_error
*err
, *unlock_err
= NULL
;
1041 struct got_lockfile
*lf
= NULL
;
1043 err
= got_object_get_path(&path
, id
, a
->repo
);
1047 err
= got_object_open_loose_fd(&fd
, id
, a
->repo
);
1051 if (fstat(fd
, &sb
) == -1) {
1052 err
= got_error_from_errno("fstat");
1057 * Do not delete objects which are younger than our maximum
1058 * modification time threshold. This prevents a race where
1059 * new objects which are being added to the repository
1060 * concurrently would be deleted.
1062 if (a
->ignore_mtime
|| sb
.st_mtime
<= a
->max_mtime
) {
1064 err
= got_lockfile_lock(&lf
, path
, -1);
1067 if (unlink(path
) == -1) {
1068 err
= got_error_from_errno2("unlink", path
);
1074 a
->size_purged
+= sb
.st_size
;
1075 if (a
->progress_cb
) {
1076 err
= a
->progress_cb(a
->progress_arg
, a
->nloose
,
1077 a
->ncommits
, a
->npurged
);
1081 if (fd
!= -1 && close(fd
) == -1 && err
== NULL
)
1082 err
= got_error_from_errno("close");
1085 unlock_err
= got_lockfile_unlock(lf
, -1);
1086 return err
? err
: unlock_err
;
1089 const struct got_error
*
1090 got_repo_purge_unreferenced_loose_objects(struct got_repository
*repo
,
1091 off_t
*size_before
, off_t
*size_after
, int *npacked
, int dry_run
,
1092 int ignore_mtime
, got_cleanup_progress_cb progress_cb
, void *progress_arg
,
1093 got_cancel_cb cancel_cb
, void *cancel_arg
)
1095 const struct got_error
*err
;
1096 struct got_object_idset
*loose_ids
;
1097 struct got_object_idset
*traversed_ids
;
1098 struct got_object_id
**referenced_ids
;
1099 int i
, nreferenced
, nloose
, ncommits
= 0;
1100 struct got_reflist_head refs
;
1101 struct got_reflist_entry
*re
;
1102 struct purge_loose_object_arg arg
;
1103 time_t max_mtime
= 0;
1111 err
= get_loose_object_ids(&loose_ids
, size_before
,
1112 progress_cb
, progress_arg
, repo
);
1115 nloose
= got_object_idset_num_elements(loose_ids
);
1117 got_object_idset_free(loose_ids
);
1121 traversed_ids
= got_object_idset_alloc();
1122 if (traversed_ids
== NULL
) {
1123 err
= got_error_from_errno("got_object_idset_alloc");
1127 err
= got_ref_list(&refs
, repo
, "", got_ref_cmp_by_name
, NULL
);
1130 if (!ignore_mtime
) {
1131 TAILQ_FOREACH(re
, &refs
, entry
) {
1132 time_t mtime
= got_ref_get_mtime(re
->ref
);
1133 if (mtime
> max_mtime
)
1137 * For safety, keep objects created within 10 minutes
1138 * before the youngest reference was created.
1140 if (max_mtime
>= 600)
1144 err
= get_reflist_object_ids(&referenced_ids
, &nreferenced
,
1145 (1 << GOT_OBJ_TYPE_COMMIT
) | (1 << GOT_OBJ_TYPE_TAG
),
1146 &refs
, repo
, cancel_cb
, cancel_arg
);
1150 for (i
= 0; i
< nreferenced
; i
++) {
1151 struct got_object_id
*id
= referenced_ids
[i
];
1152 err
= load_commit_or_tag(loose_ids
, &ncommits
, npacked
,
1153 traversed_ids
, id
, repo
, progress_cb
, progress_arg
, nloose
,
1154 cancel_cb
, cancel_arg
);
1159 /* Produce a final progress report in case no objects can be purged. */
1160 if (got_object_idset_num_elements(loose_ids
) == 0 && progress_cb
) {
1161 err
= progress_cb(progress_arg
, nloose
, ncommits
, 0);
1166 /* Any remaining loose objects are unreferenced and can be purged. */
1168 arg
.progress_arg
= progress_arg
;
1169 arg
.progress_cb
= progress_cb
;
1170 arg
.nloose
= nloose
;
1172 arg
.size_purged
= 0;
1173 arg
.ncommits
= ncommits
;
1174 arg
.dry_run
= dry_run
;
1175 arg
.max_mtime
= max_mtime
;
1176 arg
.ignore_mtime
= ignore_mtime
;
1177 err
= got_object_idset_for_each(loose_ids
, purge_loose_object
, &arg
);
1180 *size_after
= *size_before
- arg
.size_purged
;
1182 got_object_idset_free(loose_ids
);
1183 got_object_idset_free(traversed_ids
);
1187 static const struct got_error
*
1188 remove_packidx(int dir_fd
, const char *relpath
)
1190 const struct got_error
*err
, *unlock_err
;
1191 struct got_lockfile
*lf
;
1193 err
= got_lockfile_lock(&lf
, relpath
, dir_fd
);
1196 if (unlinkat(dir_fd
, relpath
, 0) == -1)
1197 err
= got_error_from_errno("unlinkat");
1198 unlock_err
= got_lockfile_unlock(lf
, dir_fd
);
1199 return err
? err
: unlock_err
;
1202 const struct got_error
*
1203 got_repo_remove_lonely_packidx(struct got_repository
*repo
, int dry_run
,
1204 got_lonely_packidx_progress_cb progress_cb
, void *progress_arg
,
1205 got_cancel_cb cancel_cb
, void *cancel_arg
)
1207 const struct got_error
*err
;
1208 DIR *packdir
= NULL
;
1209 struct dirent
*dent
;
1210 char *pack_relpath
= NULL
;
1214 packdir_fd
= openat(got_repo_get_fd(repo
),
1215 GOT_OBJECTS_PACK_DIR
, O_DIRECTORY
);
1216 if (packdir_fd
== -1) {
1217 if (errno
== ENOENT
)
1219 return got_error_from_errno_fmt("openat: %s/%s",
1220 got_repo_get_path_git_dir(repo
),
1221 GOT_OBJECTS_PACK_DIR
);
1224 packdir
= fdopendir(packdir_fd
);
1225 if (packdir
== NULL
) {
1226 err
= got_error_from_errno("fdopendir");
1230 while ((dent
= readdir(packdir
)) != NULL
) {
1232 err
= cancel_cb(cancel_arg
);
1237 if (!got_repo_is_packidx_filename(dent
->d_name
,
1238 strlen(dent
->d_name
)))
1241 err
= got_packidx_get_packfile_path(&pack_relpath
,
1246 if (fstatat(packdir_fd
, pack_relpath
, &sb
, 0) != -1) {
1248 pack_relpath
= NULL
;
1251 if (errno
!= ENOENT
) {
1252 err
= got_error_from_errno_fmt("fstatat: %s/%s/%s",
1253 got_repo_get_path_git_dir(repo
),
1254 GOT_OBJECTS_PACK_DIR
,
1260 err
= remove_packidx(packdir_fd
, dent
->d_name
);
1266 if (asprintf(&path
, "%s/%s/%s",
1267 got_repo_get_path_git_dir(repo
),
1268 GOT_OBJECTS_PACK_DIR
,
1269 dent
->d_name
) == -1) {
1270 err
= got_error_from_errno("asprintf");
1273 err
= progress_cb(progress_arg
, path
);
1279 pack_relpath
= NULL
;
1282 if (packdir
&& closedir(packdir
) != 0 && err
== NULL
)
1283 err
= got_error_from_errno("closedir");