midx.c

   1 #include "cache.h"
   2 #include "csum-file.h"
   3 #include "dir.h"
   4 #include "lockfile.h"
   5 #include "packfile.h"
   6 #include "object-store.h"
   7 #include "midx.h"
   8
   9 #define MIDX_SIGNATURE 0x4d494458 /* "MIDX" */
  10 #define MIDX_VERSION 1
  11 #define MIDX_BYTE_FILE_VERSION 4
  12 #define MIDX_BYTE_HASH_VERSION 5
  13 #define MIDX_BYTE_NUM_CHUNKS 6
  14 #define MIDX_BYTE_NUM_PACKS 8
  15 #define MIDX_HASH_VERSION 1
  16 #define MIDX_HEADER_SIZE 12
  17 #define MIDX_HASH_LEN 20
  18 #define MIDX_MIN_SIZE (MIDX_HEADER_SIZE + MIDX_HASH_LEN)
  19
  20 #define MIDX_MAX_CHUNKS 1
  21 #define MIDX_CHUNK_ALIGNMENT 4
  22 #define MIDX_CHUNKID_PACKNAMES 0x504e414d /* "PNAM" */
  23 #define MIDX_CHUNKLOOKUP_WIDTH (sizeof(uint32_t) + sizeof(uint64_t))
  24
  25 static char *get_midx_filename(const char *object_dir)
  26 {
  27         return xstrfmt("%s/pack/multi-pack-index", object_dir);
  28 }
  29
  30 struct multi_pack_index *load_multi_pack_index(const char *object_dir)
  31 {
  32         struct multi_pack_index *m = NULL;
  33         int fd;
  34         struct stat st;
  35         size_t midx_size;
  36         void *midx_map = NULL;
  37         uint32_t hash_version;
  38         char *midx_name = get_midx_filename(object_dir);
  39         uint32_t i;
  40
  41         fd = git_open(midx_name);
  42
  43         if (fd < 0)
  44                 goto cleanup_fail;
  45         if (fstat(fd, &st)) {
  46                 error_errno(_("failed to read %s"), midx_name);
  47                 goto cleanup_fail;
  48         }
  49
  50         midx_size = xsize_t(st.st_size);
  51
  52         if (midx_size < MIDX_MIN_SIZE) {
  53                 error(_("multi-pack-index file %s is too small"), midx_name);
  54                 goto cleanup_fail;
  55         }
  56
  57         FREE_AND_NULL(midx_name);
  58
  59         midx_map = xmmap(NULL, midx_size, PROT_READ, MAP_PRIVATE, fd, 0);
  60
  61         FLEX_ALLOC_MEM(m, object_dir, object_dir, strlen(object_dir));
  62         m->fd = fd;
  63         m->data = midx_map;
  64         m->data_len = midx_size;
  65
  66         m->signature = get_be32(m->data);
  67         if (m->signature != MIDX_SIGNATURE) {
  68                 error(_("multi-pack-index signature 0x%08x does not match signature 0x%08x"),
  69                       m->signature, MIDX_SIGNATURE);
  70                 goto cleanup_fail;
  71         }
  72
  73         m->version = m->data[MIDX_BYTE_FILE_VERSION];
  74         if (m->version != MIDX_VERSION) {
  75                 error(_("multi-pack-index version %d not recognized"),
  76                       m->version);
  77                 goto cleanup_fail;
  78         }
  79
  80         hash_version = m->data[MIDX_BYTE_HASH_VERSION];
  81         if (hash_version != MIDX_HASH_VERSION) {
  82                 error(_("hash version %u does not match"), hash_version);
  83                 goto cleanup_fail;
  84         }
  85         m->hash_len = MIDX_HASH_LEN;
  86
  87         m->num_chunks = m->data[MIDX_BYTE_NUM_CHUNKS];
  88
  89         m->num_packs = get_be32(m->data + MIDX_BYTE_NUM_PACKS);
  90
  91         for (i = 0; i < m->num_chunks; i++) {
  92                 uint32_t chunk_id = get_be32(m->data + MIDX_HEADER_SIZE +
  93                                              MIDX_CHUNKLOOKUP_WIDTH * i);
  94                 uint64_t chunk_offset = get_be64(m->data + MIDX_HEADER_SIZE + 4 +
  95                                                  MIDX_CHUNKLOOKUP_WIDTH * i);
  96
  97                 switch (chunk_id) {
  98                         case MIDX_CHUNKID_PACKNAMES:
  99                                 m->chunk_pack_names = m->data + chunk_offset;
 100                                 break;
 101
 102                         case 0:
 103                                 die(_("terminating multi-pack-index chunk id appears earlier than expected"));
 104                                 break;
 105
 106                         default:
 107                                 /*
 108                                  * Do nothing on unrecognized chunks, allowing future
 109                                  * extensions to add optional chunks.
 110                                  */
 111                                 break;
 112                 }
 113         }
 114
 115         if (!m->chunk_pack_names)
 116                 die(_("multi-pack-index missing required pack-name chunk"));
 117
 118         return m;
 119
 120 cleanup_fail:
 121         free(m);
 122         free(midx_name);
 123         if (midx_map)
 124                 munmap(midx_map, midx_size);
 125         if (0 <= fd)
 126                 close(fd);
 127         return NULL;
 128 }
 129
 130 static size_t write_midx_header(struct hashfile *f,
 131                                 unsigned char num_chunks,
 132                                 uint32_t num_packs)
 133 {
 134         unsigned char byte_values[4];
 135
 136         hashwrite_be32(f, MIDX_SIGNATURE);
 137         byte_values[0] = MIDX_VERSION;
 138         byte_values[1] = MIDX_HASH_VERSION;
 139         byte_values[2] = num_chunks;
 140         byte_values[3] = 0; /* unused */
 141         hashwrite(f, byte_values, sizeof(byte_values));
 142         hashwrite_be32(f, num_packs);
 143
 144         return MIDX_HEADER_SIZE;
 145 }
 146
 147 struct pack_list {
 148         struct packed_git **list;
 149         char **names;
 150         uint32_t nr;
 151         uint32_t alloc_list;
 152         uint32_t alloc_names;
 153         size_t pack_name_concat_len;
 154 };
 155
 156 static void add_pack_to_midx(const char *full_path, size_t full_path_len,
 157                              const char *file_name, void *data)
 158 {
 159         struct pack_list *packs = (struct pack_list *)data;
 160
 161         if (ends_with(file_name, ".idx")) {
 162                 ALLOC_GROW(packs->list, packs->nr + 1, packs->alloc_list);
 163                 ALLOC_GROW(packs->names, packs->nr + 1, packs->alloc_names);
 164
 165                 packs->list[packs->nr] = add_packed_git(full_path,
 166                                                         full_path_len,
 167                                                         0);
 168                 if (!packs->list[packs->nr]) {
 169                         warning(_("failed to add packfile '%s'"),
 170                                 full_path);
 171                         return;
 172                 }
 173
 174                 packs->names[packs->nr] = xstrdup(file_name);
 175                 packs->pack_name_concat_len += strlen(file_name) + 1;
 176                 packs->nr++;
 177         }
 178 }
 179
 180 struct pack_pair {
 181         uint32_t pack_int_id;
 182         char *pack_name;
 183 };
 184
 185 static int pack_pair_compare(const void *_a, const void *_b)
 186 {
 187         struct pack_pair *a = (struct pack_pair *)_a;
 188         struct pack_pair *b = (struct pack_pair *)_b;
 189         return strcmp(a->pack_name, b->pack_name);
 190 }
 191
 192 static void sort_packs_by_name(char **pack_names, uint32_t nr_packs, uint32_t *perm)
 193 {
 194         uint32_t i;
 195         struct pack_pair *pairs;
 196
 197         ALLOC_ARRAY(pairs, nr_packs);
 198
 199         for (i = 0; i < nr_packs; i++) {
 200                 pairs[i].pack_int_id = i;
 201                 pairs[i].pack_name = pack_names[i];
 202         }
 203
 204         QSORT(pairs, nr_packs, pack_pair_compare);
 205
 206         for (i = 0; i < nr_packs; i++) {
 207                 pack_names[i] = pairs[i].pack_name;
 208                 perm[pairs[i].pack_int_id] = i;
 209         }
 210
 211         free(pairs);
 212 }
 213
 214 static size_t write_midx_pack_names(struct hashfile *f,
 215                                     char **pack_names,
 216                                     uint32_t num_packs)
 217 {
 218         uint32_t i;
 219         unsigned char padding[MIDX_CHUNK_ALIGNMENT];
 220         size_t written = 0;
 221
 222         for (i = 0; i < num_packs; i++) {
 223                 size_t writelen = strlen(pack_names[i]) + 1;
 224
 225                 if (i && strcmp(pack_names[i], pack_names[i - 1]) <= 0)
 226                         BUG("incorrect pack-file order: %s before %s",
 227                             pack_names[i - 1],
 228                             pack_names[i]);
 229
 230                 hashwrite(f, pack_names[i], writelen);
 231                 written += writelen;
 232         }
 233
 234         /* add padding to be aligned */
 235         i = MIDX_CHUNK_ALIGNMENT - (written % MIDX_CHUNK_ALIGNMENT);
 236         if (i < MIDX_CHUNK_ALIGNMENT) {
 237                 memset(padding, 0, sizeof(padding));
 238                 hashwrite(f, padding, i);
 239                 written += i;
 240         }
 241
 242         return written;
 243 }
 244
 245 int write_midx_file(const char *object_dir)
 246 {
 247         unsigned char cur_chunk, num_chunks = 0;
 248         char *midx_name;
 249         uint32_t i;
 250         struct hashfile *f = NULL;
 251         struct lock_file lk;
 252         struct pack_list packs;
 253         uint32_t *pack_perm = NULL;
 254         uint64_t written = 0;
 255         uint32_t chunk_ids[MIDX_MAX_CHUNKS + 1];
 256         uint64_t chunk_offsets[MIDX_MAX_CHUNKS + 1];
 257
 258         midx_name = get_midx_filename(object_dir);
 259         if (safe_create_leading_directories(midx_name)) {
 260                 UNLEAK(midx_name);
 261                 die_errno(_("unable to create leading directories of %s"),
 262                           midx_name);
 263         }
 264
 265         packs.nr = 0;
 266         packs.alloc_list = 16;
 267         packs.alloc_names = 16;
 268         packs.list = NULL;
 269         packs.pack_name_concat_len = 0;
 270         ALLOC_ARRAY(packs.list, packs.alloc_list);
 271         ALLOC_ARRAY(packs.names, packs.alloc_names);
 272
 273         for_each_file_in_pack_dir(object_dir, add_pack_to_midx, &packs);
 274
 275         if (packs.pack_name_concat_len % MIDX_CHUNK_ALIGNMENT)
 276                 packs.pack_name_concat_len += MIDX_CHUNK_ALIGNMENT -
 277                                               (packs.pack_name_concat_len % MIDX_CHUNK_ALIGNMENT);
 278
 279         ALLOC_ARRAY(pack_perm, packs.nr);
 280         sort_packs_by_name(packs.names, packs.nr, pack_perm);
 281
 282         hold_lock_file_for_update(&lk, midx_name, LOCK_DIE_ON_ERROR);
 283         f = hashfd(lk.tempfile->fd, lk.tempfile->filename.buf);
 284         FREE_AND_NULL(midx_name);
 285
 286         cur_chunk = 0;
 287         num_chunks = 1;
 288
 289         written = write_midx_header(f, num_chunks, packs.nr);
 290
 291         chunk_ids[cur_chunk] = MIDX_CHUNKID_PACKNAMES;
 292         chunk_offsets[cur_chunk] = written + (num_chunks + 1) * MIDX_CHUNKLOOKUP_WIDTH;
 293
 294         cur_chunk++;
 295         chunk_ids[cur_chunk] = 0;
 296         chunk_offsets[cur_chunk] = chunk_offsets[cur_chunk - 1] + packs.pack_name_concat_len;
 297
 298         for (i = 0; i <= num_chunks; i++) {
 299                 if (i && chunk_offsets[i] < chunk_offsets[i - 1])
 300                         BUG("incorrect chunk offsets: %"PRIu64" before %"PRIu64,
 301                             chunk_offsets[i - 1],
 302                             chunk_offsets[i]);
 303
 304                 if (chunk_offsets[i] % MIDX_CHUNK_ALIGNMENT)
 305                         BUG("chunk offset %"PRIu64" is not properly aligned",
 306                             chunk_offsets[i]);
 307
 308                 hashwrite_be32(f, chunk_ids[i]);
 309                 hashwrite_be32(f, chunk_offsets[i] >> 32);
 310                 hashwrite_be32(f, chunk_offsets[i]);
 311
 312                 written += MIDX_CHUNKLOOKUP_WIDTH;
 313         }
 314
 315         for (i = 0; i < num_chunks; i++) {
 316                 if (written != chunk_offsets[i])
 317                         BUG("incorrect chunk offset (%"PRIu64" != %"PRIu64") for chunk id %"PRIx32,
 318                             chunk_offsets[i],
 319                             written,
 320                             chunk_ids[i]);
 321
 322                 switch (chunk_ids[i]) {
 323                         case MIDX_CHUNKID_PACKNAMES:
 324                                 written += write_midx_pack_names(f, packs.names, packs.nr);
 325                                 break;
 326
 327                         default:
 328                                 BUG("trying to write unknown chunk id %"PRIx32,
 329                                     chunk_ids[i]);
 330                 }
 331         }
 332
 333         if (written != chunk_offsets[num_chunks])
 334                 BUG("incorrect final offset %"PRIu64" != %"PRIu64,
 335                     written,
 336                     chunk_offsets[num_chunks]);
 337
 338         finalize_hashfile(f, NULL, CSUM_FSYNC | CSUM_HASH_IN_STREAM);
 339         commit_lock_file(&lk);
 340
 341         for (i = 0; i < packs.nr; i++) {
 342                 if (packs.list[i]) {
 343                         close_pack(packs.list[i]);
 344                         free(packs.list[i]);
 345                 }
 346                 free(packs.names[i]);
 347         }
 348
 349         free(packs.list);
 350         free(packs.names);
 351         return 0;
 352 }