2 Copyright 2020 Google LLC
4 Use of this source code is governed by a BSD-style
5 license that can be found in the LICENSE file or at
6 https://developers.google.com/open-source/licenses/bsd
15 #include "reftable-error.h"
16 #include "reftable-record.h"
17 #include "reftable-merged.h"
22 static int stack_try_add(struct reftable_stack
*st
,
23 int (*write_table
)(struct reftable_writer
*wr
,
26 static int stack_write_compact(struct reftable_stack
*st
,
27 struct reftable_writer
*wr
, int first
, int last
,
28 struct reftable_log_expiry_config
*config
);
29 static int stack_check_addition(struct reftable_stack
*st
,
30 const char *new_tab_name
);
31 static void reftable_addition_close(struct reftable_addition
*add
);
32 static int reftable_stack_reload_maybe_reuse(struct reftable_stack
*st
,
35 static void stack_filename(struct strbuf
*dest
, struct reftable_stack
*st
,
39 strbuf_addstr(dest
, st
->reftable_dir
);
40 strbuf_addstr(dest
, "/");
41 strbuf_addstr(dest
, name
);
44 static ssize_t
reftable_fd_write(void *arg
, const void *data
, size_t sz
)
46 int *fdp
= (int *)arg
;
47 return write_in_full(*fdp
, data
, sz
);
50 int reftable_new_stack(struct reftable_stack
**dest
, const char *dir
,
51 struct reftable_write_options config
)
53 struct reftable_stack
*p
=
54 reftable_calloc(sizeof(struct reftable_stack
));
55 struct strbuf list_file_name
= STRBUF_INIT
;
58 if (config
.hash_id
== 0) {
59 config
.hash_id
= GIT_SHA1_FORMAT_ID
;
64 strbuf_reset(&list_file_name
);
65 strbuf_addstr(&list_file_name
, dir
);
66 strbuf_addstr(&list_file_name
, "/tables.list");
68 p
->list_file
= strbuf_detach(&list_file_name
, NULL
);
70 p
->reftable_dir
= xstrdup(dir
);
73 err
= reftable_stack_reload_maybe_reuse(p
, 1);
75 reftable_stack_destroy(p
);
82 static int fd_read_lines(int fd
, char ***namesp
)
84 off_t size
= lseek(fd
, 0, SEEK_END
);
88 err
= REFTABLE_IO_ERROR
;
91 err
= lseek(fd
, 0, SEEK_SET
);
93 err
= REFTABLE_IO_ERROR
;
97 buf
= reftable_malloc(size
+ 1);
98 if (read_in_full(fd
, buf
, size
) != size
) {
99 err
= REFTABLE_IO_ERROR
;
104 parse_names(buf
, size
, namesp
);
111 int read_lines(const char *filename
, char ***namesp
)
113 int fd
= open(filename
, O_RDONLY
);
116 if (errno
== ENOENT
) {
117 *namesp
= reftable_calloc(sizeof(char *));
121 return REFTABLE_IO_ERROR
;
123 err
= fd_read_lines(fd
, namesp
);
128 struct reftable_merged_table
*
129 reftable_stack_merged_table(struct reftable_stack
*st
)
134 static int has_name(char **names
, const char *name
)
137 if (!strcmp(*names
, name
))
144 /* Close and free the stack */
145 void reftable_stack_destroy(struct reftable_stack
*st
)
150 reftable_merged_table_free(st
->merged
);
154 err
= read_lines(st
->list_file
, &names
);
156 FREE_AND_NULL(names
);
161 struct strbuf filename
= STRBUF_INIT
;
162 for (i
= 0; i
< st
->readers_len
; i
++) {
163 const char *name
= reader_name(st
->readers
[i
]);
164 strbuf_reset(&filename
);
165 if (names
&& !has_name(names
, name
)) {
166 stack_filename(&filename
, st
, name
);
168 reftable_reader_free(st
->readers
[i
]);
171 /* On Windows, can only unlink after closing. */
172 unlink(filename
.buf
);
175 strbuf_release(&filename
);
177 FREE_AND_NULL(st
->readers
);
180 if (st
->list_fd
>= 0) {
185 FREE_AND_NULL(st
->list_file
);
186 FREE_AND_NULL(st
->reftable_dir
);
191 static struct reftable_reader
**stack_copy_readers(struct reftable_stack
*st
,
194 struct reftable_reader
**cur
=
195 reftable_calloc(sizeof(struct reftable_reader
*) * cur_len
);
197 for (i
= 0; i
< cur_len
; i
++) {
198 cur
[i
] = st
->readers
[i
];
203 static int reftable_stack_reload_once(struct reftable_stack
*st
, char **names
,
206 int cur_len
= !st
->merged
? 0 : st
->merged
->stack_len
;
207 struct reftable_reader
**cur
= stack_copy_readers(st
, cur_len
);
209 int names_len
= names_length(names
);
210 struct reftable_reader
**new_readers
=
211 reftable_calloc(sizeof(struct reftable_reader
*) * names_len
);
212 struct reftable_table
*new_tables
=
213 reftable_calloc(sizeof(struct reftable_table
) * names_len
);
214 int new_readers_len
= 0;
215 struct reftable_merged_table
*new_merged
= NULL
;
216 struct strbuf table_path
= STRBUF_INIT
;
220 struct reftable_reader
*rd
= NULL
;
221 char *name
= *names
++;
223 /* this is linear; we assume compaction keeps the number of
224 tables under control so this is not quadratic. */
226 for (j
= 0; reuse_open
&& j
< cur_len
; j
++) {
227 if (cur
[j
] && 0 == strcmp(cur
[j
]->name
, name
)) {
235 struct reftable_block_source src
= { NULL
};
236 stack_filename(&table_path
, st
, name
);
238 err
= reftable_block_source_from_file(&src
,
243 err
= reftable_new_reader(&rd
, &src
, name
);
248 new_readers
[new_readers_len
] = rd
;
249 reftable_table_from_reader(&new_tables
[new_readers_len
], rd
);
254 err
= reftable_new_merged_table(&new_merged
, new_tables
,
255 new_readers_len
, st
->config
.hash_id
);
260 st
->readers_len
= new_readers_len
;
262 merged_table_release(st
->merged
);
263 reftable_merged_table_free(st
->merged
);
266 reftable_free(st
->readers
);
268 st
->readers
= new_readers
;
272 new_merged
->suppress_deletions
= 1;
273 st
->merged
= new_merged
;
274 for (i
= 0; i
< cur_len
; i
++) {
276 const char *name
= reader_name(cur
[i
]);
277 stack_filename(&table_path
, st
, name
);
279 reader_close(cur
[i
]);
280 reftable_reader_free(cur
[i
]);
282 /* On Windows, can only unlink after closing. */
283 unlink(table_path
.buf
);
288 for (i
= 0; i
< new_readers_len
; i
++) {
289 reader_close(new_readers
[i
]);
290 reftable_reader_free(new_readers
[i
]);
292 reftable_free(new_readers
);
293 reftable_free(new_tables
);
295 strbuf_release(&table_path
);
299 /* return negative if a before b. */
300 static int tv_cmp(struct timeval
*a
, struct timeval
*b
)
302 time_t diff
= a
->tv_sec
- b
->tv_sec
;
303 int udiff
= a
->tv_usec
- b
->tv_usec
;
311 static int reftable_stack_reload_maybe_reuse(struct reftable_stack
*st
,
314 char **names
= NULL
, **names_after
= NULL
;
315 struct timeval deadline
;
320 err
= gettimeofday(&deadline
, NULL
);
323 deadline
.tv_sec
+= 3;
328 err
= gettimeofday(&now
, NULL
);
333 * Only look at deadlines after the first few times. This
334 * simplifies debugging in GDB.
337 if (tries
> 3 && tv_cmp(&now
, &deadline
) >= 0)
340 fd
= open(st
->list_file
, O_RDONLY
);
342 if (errno
!= ENOENT
) {
343 err
= REFTABLE_IO_ERROR
;
347 names
= reftable_calloc(sizeof(char *));
349 err
= fd_read_lines(fd
, &names
);
354 err
= reftable_stack_reload_once(st
, names
, reuse_open
);
357 if (err
!= REFTABLE_NOT_EXIST_ERROR
)
361 * REFTABLE_NOT_EXIST_ERROR can be caused by a concurrent
362 * writer. Check if there was one by checking if the name list
365 err
= read_lines(st
->list_file
, &names_after
);
368 if (names_equal(names_after
, names
)) {
369 err
= REFTABLE_NOT_EXIST_ERROR
;
375 free_names(names_after
);
380 delay
= delay
+ (delay
* rand()) / RAND_MAX
+ 1;
381 sleep_millisec(delay
);
386 * Invalidate the stat cache. It is sufficient to only close the file
387 * descriptor and keep the cached stat info because we never use the
388 * latter when the former is negative.
390 if (st
->list_fd
>= 0) {
396 * Cache stat information in case it provides a useful signal to us.
397 * According to POSIX, "The st_ino and st_dev fields taken together
398 * uniquely identify the file within the system." That being said,
399 * Windows is not POSIX compliant and we do not have these fields
400 * available. So the information we have there is insufficient to
401 * determine whether two file descriptors point to the same file.
403 * While we could fall back to using other signals like the file's
404 * mtime, those are not sufficient to avoid races. We thus refrain from
405 * using the stat cache on such systems and fall back to the secondary
406 * caching mechanism, which is to check whether contents of the file
409 * On other systems which are POSIX compliant we must keep the file
410 * descriptor open. This is to avoid a race condition where two
411 * processes access the reftable stack at the same point in time:
413 * 1. A reads the reftable stack and caches its stat info.
415 * 2. B updates the stack, appending a new table to "tables.list".
416 * This will both use a new inode and result in a different file
417 * size, thus invalidating A's cache in theory.
419 * 3. B decides to auto-compact the stack and merges two tables. The
420 * file size now matches what A has cached again. Furthermore, the
421 * filesystem may decide to recycle the inode number of the file
422 * we have replaced in (2) because it is not in use anymore.
424 * 4. A reloads the reftable stack. Neither the inode number nor the
425 * file size changed. If the timestamps did not change either then
426 * we think the cached copy of our stack is up-to-date.
428 * By keeping the file descriptor open the inode number cannot be
429 * recycled, mitigating the race.
431 if (!err
&& fd
>= 0 && !fstat(fd
, &st
->list_st
) &&
432 st
->list_st
.st_dev
&& st
->list_st
.st_ino
) {
440 free_names(names_after
);
447 static int stack_uptodate(struct reftable_stack
*st
)
454 * When we have cached stat information available then we use it to
455 * verify whether the file has been rewritten.
457 * Note that we explicitly do not want to use `stat_validity_check()`
458 * and friends here because they may end up not comparing the `st_dev`
459 * and `st_ino` fields. These functions thus cannot guarantee that we
460 * indeed still have the same file.
462 if (st
->list_fd
>= 0) {
465 if (stat(st
->list_file
, &list_st
) < 0) {
467 * It's fine for "tables.list" to not exist. In that
468 * case, we have to refresh when the loaded stack has
472 return !!st
->readers_len
;
473 return REFTABLE_IO_ERROR
;
477 * When "tables.list" refers to the same file we can assume
478 * that it didn't change. This is because we always use
479 * rename(3P) to update the file and never write to it
482 if (st
->list_st
.st_dev
== list_st
.st_dev
&&
483 st
->list_st
.st_ino
== list_st
.st_ino
)
487 err
= read_lines(st
->list_file
, &names
);
491 for (i
= 0; i
< st
->readers_len
; i
++) {
497 if (strcmp(st
->readers
[i
]->name
, names
[i
])) {
503 if (names
[st
->merged
->stack_len
]) {
513 int reftable_stack_reload(struct reftable_stack
*st
)
515 int err
= stack_uptodate(st
);
517 return reftable_stack_reload_maybe_reuse(st
, 1);
521 int reftable_stack_add(struct reftable_stack
*st
,
522 int (*write
)(struct reftable_writer
*wr
, void *arg
),
525 int err
= stack_try_add(st
, write
, arg
);
527 if (err
== REFTABLE_LOCK_ERROR
) {
528 /* Ignore error return, we want to propagate
531 reftable_stack_reload(st
);
536 if (!st
->disable_auto_compact
)
537 return reftable_stack_auto_compact(st
);
542 static void format_name(struct strbuf
*dest
, uint64_t min
, uint64_t max
)
545 uint32_t rnd
= (uint32_t)git_rand();
546 snprintf(buf
, sizeof(buf
), "0x%012" PRIx64
"-0x%012" PRIx64
"-%08x",
549 strbuf_addstr(dest
, buf
);
552 struct reftable_addition
{
553 struct tempfile
*lock_file
;
554 struct reftable_stack
*stack
;
558 uint64_t next_update_index
;
561 #define REFTABLE_ADDITION_INIT {0}
563 static int reftable_stack_init_addition(struct reftable_addition
*add
,
564 struct reftable_stack
*st
)
566 struct strbuf lock_file_name
= STRBUF_INIT
;
570 strbuf_addf(&lock_file_name
, "%s.lock", st
->list_file
);
572 add
->lock_file
= create_tempfile(lock_file_name
.buf
);
573 if (!add
->lock_file
) {
574 if (errno
== EEXIST
) {
575 err
= REFTABLE_LOCK_ERROR
;
577 err
= REFTABLE_IO_ERROR
;
581 if (st
->config
.default_permissions
) {
582 if (chmod(add
->lock_file
->filename
.buf
, st
->config
.default_permissions
) < 0) {
583 err
= REFTABLE_IO_ERROR
;
588 err
= stack_uptodate(st
);
593 err
= REFTABLE_LOCK_ERROR
;
597 add
->next_update_index
= reftable_stack_next_update_index(st
);
600 reftable_addition_close(add
);
602 strbuf_release(&lock_file_name
);
606 static void reftable_addition_close(struct reftable_addition
*add
)
609 struct strbuf nm
= STRBUF_INIT
;
610 for (i
= 0; i
< add
->new_tables_len
; i
++) {
611 stack_filename(&nm
, add
->stack
, add
->new_tables
[i
]);
613 reftable_free(add
->new_tables
[i
]);
614 add
->new_tables
[i
] = NULL
;
616 reftable_free(add
->new_tables
);
617 add
->new_tables
= NULL
;
618 add
->new_tables_len
= 0;
620 delete_tempfile(&add
->lock_file
);
624 void reftable_addition_destroy(struct reftable_addition
*add
)
629 reftable_addition_close(add
);
633 int reftable_addition_commit(struct reftable_addition
*add
)
635 struct strbuf table_list
= STRBUF_INIT
;
636 int lock_file_fd
= get_tempfile_fd(add
->lock_file
);
640 if (add
->new_tables_len
== 0)
643 for (i
= 0; i
< add
->stack
->merged
->stack_len
; i
++) {
644 strbuf_addstr(&table_list
, add
->stack
->readers
[i
]->name
);
645 strbuf_addstr(&table_list
, "\n");
647 for (i
= 0; i
< add
->new_tables_len
; i
++) {
648 strbuf_addstr(&table_list
, add
->new_tables
[i
]);
649 strbuf_addstr(&table_list
, "\n");
652 err
= write_in_full(lock_file_fd
, table_list
.buf
, table_list
.len
);
653 strbuf_release(&table_list
);
655 err
= REFTABLE_IO_ERROR
;
659 err
= rename_tempfile(&add
->lock_file
, add
->stack
->list_file
);
661 err
= REFTABLE_IO_ERROR
;
665 /* success, no more state to clean up. */
666 for (i
= 0; i
< add
->new_tables_len
; i
++) {
667 reftable_free(add
->new_tables
[i
]);
669 reftable_free(add
->new_tables
);
670 add
->new_tables
= NULL
;
671 add
->new_tables_len
= 0;
673 err
= reftable_stack_reload_maybe_reuse(add
->stack
, 1);
677 if (!add
->stack
->disable_auto_compact
)
678 err
= reftable_stack_auto_compact(add
->stack
);
681 reftable_addition_close(add
);
685 int reftable_stack_new_addition(struct reftable_addition
**dest
,
686 struct reftable_stack
*st
)
689 struct reftable_addition empty
= REFTABLE_ADDITION_INIT
;
690 *dest
= reftable_calloc(sizeof(**dest
));
692 err
= reftable_stack_init_addition(*dest
, st
);
694 reftable_free(*dest
);
700 static int stack_try_add(struct reftable_stack
*st
,
701 int (*write_table
)(struct reftable_writer
*wr
,
705 struct reftable_addition add
= REFTABLE_ADDITION_INIT
;
706 int err
= reftable_stack_init_addition(&add
, st
);
710 err
= REFTABLE_LOCK_ERROR
;
714 err
= reftable_addition_add(&add
, write_table
, arg
);
718 err
= reftable_addition_commit(&add
);
720 reftable_addition_close(&add
);
724 int reftable_addition_add(struct reftable_addition
*add
,
725 int (*write_table
)(struct reftable_writer
*wr
,
729 struct strbuf temp_tab_file_name
= STRBUF_INIT
;
730 struct strbuf tab_file_name
= STRBUF_INIT
;
731 struct strbuf next_name
= STRBUF_INIT
;
732 struct reftable_writer
*wr
= NULL
;
736 strbuf_reset(&next_name
);
737 format_name(&next_name
, add
->next_update_index
, add
->next_update_index
);
739 stack_filename(&temp_tab_file_name
, add
->stack
, next_name
.buf
);
740 strbuf_addstr(&temp_tab_file_name
, ".temp.XXXXXX");
742 tab_fd
= mkstemp(temp_tab_file_name
.buf
);
744 err
= REFTABLE_IO_ERROR
;
747 if (add
->stack
->config
.default_permissions
) {
748 if (chmod(temp_tab_file_name
.buf
, add
->stack
->config
.default_permissions
)) {
749 err
= REFTABLE_IO_ERROR
;
753 wr
= reftable_new_writer(reftable_fd_write
, &tab_fd
,
754 &add
->stack
->config
);
755 err
= write_table(wr
, arg
);
759 err
= reftable_writer_close(wr
);
760 if (err
== REFTABLE_EMPTY_TABLE_ERROR
) {
770 err
= REFTABLE_IO_ERROR
;
774 err
= stack_check_addition(add
->stack
, temp_tab_file_name
.buf
);
778 if (wr
->min_update_index
< add
->next_update_index
) {
779 err
= REFTABLE_API_ERROR
;
783 format_name(&next_name
, wr
->min_update_index
, wr
->max_update_index
);
784 strbuf_addstr(&next_name
, ".ref");
786 stack_filename(&tab_file_name
, add
->stack
, next_name
.buf
);
789 On windows, this relies on rand() picking a unique destination name.
790 Maybe we should do retry loop as well?
792 err
= rename(temp_tab_file_name
.buf
, tab_file_name
.buf
);
794 err
= REFTABLE_IO_ERROR
;
798 add
->new_tables
= reftable_realloc(add
->new_tables
,
799 sizeof(*add
->new_tables
) *
800 (add
->new_tables_len
+ 1));
801 add
->new_tables
[add
->new_tables_len
] = strbuf_detach(&next_name
, NULL
);
802 add
->new_tables_len
++;
808 if (temp_tab_file_name
.len
> 0) {
809 unlink(temp_tab_file_name
.buf
);
812 strbuf_release(&temp_tab_file_name
);
813 strbuf_release(&tab_file_name
);
814 strbuf_release(&next_name
);
815 reftable_writer_free(wr
);
819 uint64_t reftable_stack_next_update_index(struct reftable_stack
*st
)
821 int sz
= st
->merged
->stack_len
;
823 return reftable_reader_max_update_index(st
->readers
[sz
- 1]) +
828 static int stack_compact_locked(struct reftable_stack
*st
, int first
, int last
,
829 struct strbuf
*temp_tab
,
830 struct reftable_log_expiry_config
*config
)
832 struct strbuf next_name
= STRBUF_INIT
;
834 struct reftable_writer
*wr
= NULL
;
837 format_name(&next_name
,
838 reftable_reader_min_update_index(st
->readers
[first
]),
839 reftable_reader_max_update_index(st
->readers
[last
]));
841 stack_filename(temp_tab
, st
, next_name
.buf
);
842 strbuf_addstr(temp_tab
, ".temp.XXXXXX");
844 tab_fd
= mkstemp(temp_tab
->buf
);
845 wr
= reftable_new_writer(reftable_fd_write
, &tab_fd
, &st
->config
);
847 err
= stack_write_compact(st
, wr
, first
, last
, config
);
850 err
= reftable_writer_close(wr
);
858 reftable_writer_free(wr
);
863 if (err
!= 0 && temp_tab
->len
> 0) {
864 unlink(temp_tab
->buf
);
865 strbuf_release(temp_tab
);
867 strbuf_release(&next_name
);
871 static int stack_write_compact(struct reftable_stack
*st
,
872 struct reftable_writer
*wr
, int first
, int last
,
873 struct reftable_log_expiry_config
*config
)
875 int subtabs_len
= last
- first
+ 1;
876 struct reftable_table
*subtabs
= reftable_calloc(
877 sizeof(struct reftable_table
) * (last
- first
+ 1));
878 struct reftable_merged_table
*mt
= NULL
;
880 struct reftable_iterator it
= { NULL
};
881 struct reftable_ref_record ref
= { NULL
};
882 struct reftable_log_record log
= { NULL
};
884 uint64_t entries
= 0;
887 for (i
= first
, j
= 0; i
<= last
; i
++) {
888 struct reftable_reader
*t
= st
->readers
[i
];
889 reftable_table_from_reader(&subtabs
[j
++], t
);
890 st
->stats
.bytes
+= t
->size
;
892 reftable_writer_set_limits(wr
, st
->readers
[first
]->min_update_index
,
893 st
->readers
[last
]->max_update_index
);
895 err
= reftable_new_merged_table(&mt
, subtabs
, subtabs_len
,
898 reftable_free(subtabs
);
902 err
= reftable_merged_table_seek_ref(mt
, &it
, "");
907 err
= reftable_iterator_next_ref(&it
, &ref
);
916 if (first
== 0 && reftable_ref_record_is_deletion(&ref
)) {
920 err
= reftable_writer_add_ref(wr
, &ref
);
926 reftable_iterator_destroy(&it
);
928 err
= reftable_merged_table_seek_log(mt
, &it
, "");
933 err
= reftable_iterator_next_log(&it
, &log
);
941 if (first
== 0 && reftable_log_record_is_deletion(&log
)) {
945 if (config
&& config
->min_update_index
> 0 &&
946 log
.update_index
< config
->min_update_index
) {
950 if (config
&& config
->time
> 0 &&
951 log
.value
.update
.time
< config
->time
) {
955 err
= reftable_writer_add_log(wr
, &log
);
963 reftable_iterator_destroy(&it
);
965 merged_table_release(mt
);
966 reftable_merged_table_free(mt
);
968 reftable_ref_record_release(&ref
);
969 reftable_log_record_release(&log
);
970 st
->stats
.entries_written
+= entries
;
974 /* < 0: error. 0 == OK, > 0 attempt failed; could retry. */
975 static int stack_compact_range(struct reftable_stack
*st
, int first
, int last
,
976 struct reftable_log_expiry_config
*expiry
)
978 struct strbuf temp_tab_file_name
= STRBUF_INIT
;
979 struct strbuf new_table_name
= STRBUF_INIT
;
980 struct strbuf lock_file_name
= STRBUF_INIT
;
981 struct strbuf ref_list_contents
= STRBUF_INIT
;
982 struct strbuf new_table_path
= STRBUF_INIT
;
985 int lock_file_fd
= -1;
986 int compact_count
= last
- first
+ 1;
988 char **delete_on_success
=
989 reftable_calloc(sizeof(char *) * (compact_count
+ 1));
990 char **subtable_locks
=
991 reftable_calloc(sizeof(char *) * (compact_count
+ 1));
994 int is_empty_table
= 0;
996 if (first
> last
|| (!expiry
&& first
== last
)) {
1001 st
->stats
.attempts
++;
1003 strbuf_reset(&lock_file_name
);
1004 strbuf_addstr(&lock_file_name
, st
->list_file
);
1005 strbuf_addstr(&lock_file_name
, ".lock");
1008 open(lock_file_name
.buf
, O_EXCL
| O_CREAT
| O_WRONLY
, 0666);
1009 if (lock_file_fd
< 0) {
1010 if (errno
== EEXIST
) {
1013 err
= REFTABLE_IO_ERROR
;
1017 /* Don't want to write to the lock for now. */
1018 close(lock_file_fd
);
1022 err
= stack_uptodate(st
);
1026 for (i
= first
, j
= 0; i
<= last
; i
++) {
1027 struct strbuf subtab_file_name
= STRBUF_INIT
;
1028 struct strbuf subtab_lock
= STRBUF_INIT
;
1029 int sublock_file_fd
= -1;
1031 stack_filename(&subtab_file_name
, st
,
1032 reader_name(st
->readers
[i
]));
1034 strbuf_reset(&subtab_lock
);
1035 strbuf_addbuf(&subtab_lock
, &subtab_file_name
);
1036 strbuf_addstr(&subtab_lock
, ".lock");
1038 sublock_file_fd
= open(subtab_lock
.buf
,
1039 O_EXCL
| O_CREAT
| O_WRONLY
, 0666);
1040 if (sublock_file_fd
>= 0) {
1041 close(sublock_file_fd
);
1042 } else if (sublock_file_fd
< 0) {
1043 if (errno
== EEXIST
) {
1046 err
= REFTABLE_IO_ERROR
;
1050 subtable_locks
[j
] = subtab_lock
.buf
;
1051 delete_on_success
[j
] = subtab_file_name
.buf
;
1058 err
= unlink(lock_file_name
.buf
);
1063 err
= stack_compact_locked(st
, first
, last
, &temp_tab_file_name
,
1065 /* Compaction + tombstones can create an empty table out of non-empty
1067 is_empty_table
= (err
== REFTABLE_EMPTY_TABLE_ERROR
);
1068 if (is_empty_table
) {
1075 open(lock_file_name
.buf
, O_EXCL
| O_CREAT
| O_WRONLY
, 0666);
1076 if (lock_file_fd
< 0) {
1077 if (errno
== EEXIST
) {
1080 err
= REFTABLE_IO_ERROR
;
1085 if (st
->config
.default_permissions
) {
1086 if (chmod(lock_file_name
.buf
, st
->config
.default_permissions
) < 0) {
1087 err
= REFTABLE_IO_ERROR
;
1092 format_name(&new_table_name
, st
->readers
[first
]->min_update_index
,
1093 st
->readers
[last
]->max_update_index
);
1094 strbuf_addstr(&new_table_name
, ".ref");
1096 stack_filename(&new_table_path
, st
, new_table_name
.buf
);
1098 if (!is_empty_table
) {
1100 err
= rename(temp_tab_file_name
.buf
, new_table_path
.buf
);
1102 err
= REFTABLE_IO_ERROR
;
1107 for (i
= 0; i
< first
; i
++) {
1108 strbuf_addstr(&ref_list_contents
, st
->readers
[i
]->name
);
1109 strbuf_addstr(&ref_list_contents
, "\n");
1111 if (!is_empty_table
) {
1112 strbuf_addbuf(&ref_list_contents
, &new_table_name
);
1113 strbuf_addstr(&ref_list_contents
, "\n");
1115 for (i
= last
+ 1; i
< st
->merged
->stack_len
; i
++) {
1116 strbuf_addstr(&ref_list_contents
, st
->readers
[i
]->name
);
1117 strbuf_addstr(&ref_list_contents
, "\n");
1120 err
= write_in_full(lock_file_fd
, ref_list_contents
.buf
, ref_list_contents
.len
);
1122 err
= REFTABLE_IO_ERROR
;
1123 unlink(new_table_path
.buf
);
1126 err
= close(lock_file_fd
);
1129 err
= REFTABLE_IO_ERROR
;
1130 unlink(new_table_path
.buf
);
1134 err
= rename(lock_file_name
.buf
, st
->list_file
);
1136 err
= REFTABLE_IO_ERROR
;
1137 unlink(new_table_path
.buf
);
1142 /* Reload the stack before deleting. On windows, we can only delete the
1143 files after we closed them.
1145 err
= reftable_stack_reload_maybe_reuse(st
, first
< last
);
1147 listp
= delete_on_success
;
1149 if (strcmp(*listp
, new_table_path
.buf
)) {
1156 free_names(delete_on_success
);
1158 listp
= subtable_locks
;
1163 free_names(subtable_locks
);
1164 if (lock_file_fd
>= 0) {
1165 close(lock_file_fd
);
1169 unlink(lock_file_name
.buf
);
1171 strbuf_release(&new_table_name
);
1172 strbuf_release(&new_table_path
);
1173 strbuf_release(&ref_list_contents
);
1174 strbuf_release(&temp_tab_file_name
);
1175 strbuf_release(&lock_file_name
);
1179 int reftable_stack_compact_all(struct reftable_stack
*st
,
1180 struct reftable_log_expiry_config
*config
)
1182 return stack_compact_range(st
, 0, st
->merged
->stack_len
- 1, config
);
1185 static int stack_compact_range_stats(struct reftable_stack
*st
, int first
,
1187 struct reftable_log_expiry_config
*config
)
1189 int err
= stack_compact_range(st
, first
, last
, config
);
1191 st
->stats
.failures
++;
1196 static int segment_size(struct segment
*s
)
1198 return s
->end
- s
->start
;
1201 int fastlog2(uint64_t sz
)
1206 for (; sz
; sz
/= 2) {
1212 struct segment
*sizes_to_segments(int *seglen
, uint64_t *sizes
, int n
)
1214 struct segment
*segs
= reftable_calloc(sizeof(struct segment
) * n
);
1216 struct segment cur
= { 0 };
1223 for (i
= 0; i
< n
; i
++) {
1224 int log
= fastlog2(sizes
[i
]);
1225 if (cur
.log
!= log
&& cur
.bytes
> 0) {
1226 struct segment fresh
= {
1236 cur
.bytes
+= sizes
[i
];
1243 struct segment
suggest_compaction_segment(uint64_t *sizes
, int n
)
1246 struct segment
*segs
= sizes_to_segments(&seglen
, sizes
, n
);
1247 struct segment min_seg
= {
1251 for (i
= 0; i
< seglen
; i
++) {
1252 if (segment_size(&segs
[i
]) == 1) {
1256 if (segs
[i
].log
< min_seg
.log
) {
1261 while (min_seg
.start
> 0) {
1262 int prev
= min_seg
.start
- 1;
1263 if (fastlog2(min_seg
.bytes
) < fastlog2(sizes
[prev
])) {
1267 min_seg
.start
= prev
;
1268 min_seg
.bytes
+= sizes
[prev
];
1271 reftable_free(segs
);
1275 static uint64_t *stack_table_sizes_for_compaction(struct reftable_stack
*st
)
1278 reftable_calloc(sizeof(uint64_t) * st
->merged
->stack_len
);
1279 int version
= (st
->config
.hash_id
== GIT_SHA1_FORMAT_ID
) ? 1 : 2;
1280 int overhead
= header_size(version
) - 1;
1282 for (i
= 0; i
< st
->merged
->stack_len
; i
++) {
1283 sizes
[i
] = st
->readers
[i
]->size
- overhead
;
1288 int reftable_stack_auto_compact(struct reftable_stack
*st
)
1290 uint64_t *sizes
= stack_table_sizes_for_compaction(st
);
1291 struct segment seg
=
1292 suggest_compaction_segment(sizes
, st
->merged
->stack_len
);
1293 reftable_free(sizes
);
1294 if (segment_size(&seg
) > 0)
1295 return stack_compact_range_stats(st
, seg
.start
, seg
.end
- 1,
1301 struct reftable_compaction_stats
*
1302 reftable_stack_compaction_stats(struct reftable_stack
*st
)
1307 int reftable_stack_read_ref(struct reftable_stack
*st
, const char *refname
,
1308 struct reftable_ref_record
*ref
)
1310 struct reftable_table tab
= { NULL
};
1311 reftable_table_from_merged_table(&tab
, reftable_stack_merged_table(st
));
1312 return reftable_table_read_ref(&tab
, refname
, ref
);
1315 int reftable_stack_read_log(struct reftable_stack
*st
, const char *refname
,
1316 struct reftable_log_record
*log
)
1318 struct reftable_iterator it
= { NULL
};
1319 struct reftable_merged_table
*mt
= reftable_stack_merged_table(st
);
1320 int err
= reftable_merged_table_seek_log(mt
, &it
, refname
);
1324 err
= reftable_iterator_next_log(&it
, log
);
1328 if (strcmp(log
->refname
, refname
) ||
1329 reftable_log_record_is_deletion(log
)) {
1336 reftable_log_record_release(log
);
1338 reftable_iterator_destroy(&it
);
1342 static int stack_check_addition(struct reftable_stack
*st
,
1343 const char *new_tab_name
)
1346 struct reftable_block_source src
= { NULL
};
1347 struct reftable_reader
*rd
= NULL
;
1348 struct reftable_table tab
= { NULL
};
1349 struct reftable_ref_record
*refs
= NULL
;
1350 struct reftable_iterator it
= { NULL
};
1355 if (st
->config
.skip_name_check
)
1358 err
= reftable_block_source_from_file(&src
, new_tab_name
);
1362 err
= reftable_new_reader(&rd
, &src
, new_tab_name
);
1366 err
= reftable_reader_seek_ref(rd
, &it
, "");
1375 struct reftable_ref_record ref
= { NULL
};
1376 err
= reftable_iterator_next_ref(&it
, &ref
);
1385 refs
= reftable_realloc(refs
, cap
* sizeof(refs
[0]));
1391 reftable_table_from_merged_table(&tab
, reftable_stack_merged_table(st
));
1393 err
= validate_ref_record_addition(tab
, refs
, len
);
1396 for (i
= 0; i
< len
; i
++) {
1397 reftable_ref_record_release(&refs
[i
]);
1401 reftable_iterator_destroy(&it
);
1402 reftable_reader_free(rd
);
1406 static int is_table_name(const char *s
)
1408 const char *dot
= strrchr(s
, '.');
1409 return dot
&& !strcmp(dot
, ".ref");
1412 static void remove_maybe_stale_table(struct reftable_stack
*st
, uint64_t max
,
1416 uint64_t update_idx
= 0;
1417 struct reftable_block_source src
= { NULL
};
1418 struct reftable_reader
*rd
= NULL
;
1419 struct strbuf table_path
= STRBUF_INIT
;
1420 stack_filename(&table_path
, st
, name
);
1422 err
= reftable_block_source_from_file(&src
, table_path
.buf
);
1426 err
= reftable_new_reader(&rd
, &src
, name
);
1430 update_idx
= reftable_reader_max_update_index(rd
);
1431 reftable_reader_free(rd
);
1433 if (update_idx
<= max
) {
1434 unlink(table_path
.buf
);
1437 strbuf_release(&table_path
);
1440 static int reftable_stack_clean_locked(struct reftable_stack
*st
)
1442 uint64_t max
= reftable_merged_table_max_update_index(
1443 reftable_stack_merged_table(st
));
1444 DIR *dir
= opendir(st
->reftable_dir
);
1445 struct dirent
*d
= NULL
;
1447 return REFTABLE_IO_ERROR
;
1450 while ((d
= readdir(dir
))) {
1453 if (!is_table_name(d
->d_name
))
1456 for (i
= 0; !found
&& i
< st
->readers_len
; i
++) {
1457 found
= !strcmp(reader_name(st
->readers
[i
]), d
->d_name
);
1462 remove_maybe_stale_table(st
, max
, d
->d_name
);
1469 int reftable_stack_clean(struct reftable_stack
*st
)
1471 struct reftable_addition
*add
= NULL
;
1472 int err
= reftable_stack_new_addition(&add
, st
);
1477 err
= reftable_stack_reload(st
);
1482 err
= reftable_stack_clean_locked(st
);
1485 reftable_addition_destroy(add
);
1489 int reftable_stack_print_directory(const char *stackdir
, uint32_t hash_id
)
1491 struct reftable_stack
*stack
= NULL
;
1492 struct reftable_write_options cfg
= { .hash_id
= hash_id
};
1493 struct reftable_merged_table
*merged
= NULL
;
1494 struct reftable_table table
= { NULL
};
1496 int err
= reftable_new_stack(&stack
, stackdir
, cfg
);
1500 merged
= reftable_stack_merged_table(stack
);
1501 reftable_table_from_merged_table(&table
, merged
);
1502 err
= reftable_table_print(&table
);
1505 reftable_stack_destroy(stack
);