Sync with maint
[git.git] / name-hash.c
blobfb526a3775f69eb02d6b7e47629bf60a8a0153fa
1 /*
2 * name-hash.c
4 * Hashing names in the index state
6 * Copyright (C) 2008 Linus Torvalds
7 */
8 #include "cache.h"
9 #include "thread-utils.h"
11 struct dir_entry {
12 struct hashmap_entry ent;
13 struct dir_entry *parent;
14 int nr;
15 unsigned int namelen;
16 char name[FLEX_ARRAY];
19 static int dir_entry_cmp(const void *unused_cmp_data,
20 const struct hashmap_entry *eptr,
21 const struct hashmap_entry *entry_or_key,
22 const void *keydata)
24 const struct dir_entry *e1, *e2;
25 const char *name = keydata;
27 e1 = container_of(eptr, const struct dir_entry, ent);
28 e2 = container_of(entry_or_key, const struct dir_entry, ent);
30 return e1->namelen != e2->namelen || strncasecmp(e1->name,
31 name ? name : e2->name, e1->namelen);
34 static struct dir_entry *find_dir_entry__hash(struct index_state *istate,
35 const char *name, unsigned int namelen, unsigned int hash)
37 struct dir_entry key;
38 hashmap_entry_init(&key.ent, hash);
39 key.namelen = namelen;
40 return hashmap_get_entry(&istate->dir_hash, &key, ent, name);
43 static struct dir_entry *find_dir_entry(struct index_state *istate,
44 const char *name, unsigned int namelen)
46 return find_dir_entry__hash(istate, name, namelen, memihash(name, namelen));
49 static struct dir_entry *hash_dir_entry(struct index_state *istate,
50 struct cache_entry *ce, int namelen)
53 * Throw each directory component in the hash for quick lookup
54 * during a git status. Directory components are stored without their
55 * closing slash. Despite submodules being a directory, they never
56 * reach this point, because they are stored
57 * in index_state.name_hash (as ordinary cache_entries).
59 struct dir_entry *dir;
61 /* get length of parent directory */
62 while (namelen > 0 && !is_dir_sep(ce->name[namelen - 1]))
63 namelen--;
64 if (namelen <= 0)
65 return NULL;
66 namelen--;
68 /* lookup existing entry for that directory */
69 dir = find_dir_entry(istate, ce->name, namelen);
70 if (!dir) {
71 /* not found, create it and add to hash table */
72 FLEX_ALLOC_MEM(dir, name, ce->name, namelen);
73 hashmap_entry_init(&dir->ent, memihash(ce->name, namelen));
74 dir->namelen = namelen;
75 hashmap_add(&istate->dir_hash, &dir->ent);
77 /* recursively add missing parent directories */
78 dir->parent = hash_dir_entry(istate, ce, namelen);
80 return dir;
83 static void add_dir_entry(struct index_state *istate, struct cache_entry *ce)
85 /* Add reference to the directory entry (and parents if 0). */
86 struct dir_entry *dir = hash_dir_entry(istate, ce, ce_namelen(ce));
87 while (dir && !(dir->nr++))
88 dir = dir->parent;
91 static void remove_dir_entry(struct index_state *istate, struct cache_entry *ce)
94 * Release reference to the directory entry. If 0, remove and continue
95 * with parent directory.
97 struct dir_entry *dir = hash_dir_entry(istate, ce, ce_namelen(ce));
98 while (dir && !(--dir->nr)) {
99 struct dir_entry *parent = dir->parent;
100 hashmap_remove(&istate->dir_hash, &dir->ent, NULL);
101 free(dir);
102 dir = parent;
106 static void hash_index_entry(struct index_state *istate, struct cache_entry *ce)
108 if (ce->ce_flags & CE_HASHED)
109 return;
110 ce->ce_flags |= CE_HASHED;
111 hashmap_entry_init(&ce->ent, memihash(ce->name, ce_namelen(ce)));
112 hashmap_add(&istate->name_hash, &ce->ent);
114 if (ignore_case)
115 add_dir_entry(istate, ce);
118 static int cache_entry_cmp(const void *unused_cmp_data,
119 const struct hashmap_entry *eptr,
120 const struct hashmap_entry *entry_or_key,
121 const void *remove)
123 const struct cache_entry *ce1, *ce2;
125 ce1 = container_of(eptr, const struct cache_entry, ent);
126 ce2 = container_of(entry_or_key, const struct cache_entry, ent);
129 * For remove_name_hash, find the exact entry (pointer equality); for
130 * index_file_exists, find all entries with matching hash code and
131 * decide whether the entry matches in same_name.
133 return remove ? !(ce1 == ce2) : 0;
136 static int lazy_try_threaded = 1;
137 static int lazy_nr_dir_threads;
140 * Set a minimum number of cache_entries that we will handle per
141 * thread and use that to decide how many threads to run (up to
142 * the number on the system).
144 * For guidance setting the lower per-thread bound, see:
145 * t/helper/test-lazy-init-name-hash --analyze
147 #define LAZY_THREAD_COST (2000)
150 * We use n mutexes to guard n partitions of the "istate->dir_hash"
151 * hashtable. Since "find" and "insert" operations will hash to a
152 * particular bucket and modify/search a single chain, we can say
153 * that "all chains mod n" are guarded by the same mutex -- rather
154 * than having a single mutex to guard the entire table. (This does
155 * require that we disable "rehashing" on the hashtable.)
157 * So, a larger value here decreases the probability of a collision
158 * and the time that each thread must wait for the mutex.
160 #define LAZY_MAX_MUTEX (32)
162 static pthread_mutex_t *lazy_dir_mutex_array;
165 * An array of lazy_entry items is used by the n threads in
166 * the directory parse (first) phase to (lock-free) store the
167 * intermediate results. These values are then referenced by
168 * the 2 threads in the second phase.
170 struct lazy_entry {
171 struct dir_entry *dir;
172 unsigned int hash_dir;
173 unsigned int hash_name;
177 * Decide if we want to use threads (if available) to load
178 * the hash tables. We set "lazy_nr_dir_threads" to zero when
179 * it is not worth it.
181 static int lookup_lazy_params(struct index_state *istate)
183 int nr_cpus;
185 lazy_nr_dir_threads = 0;
187 if (!lazy_try_threaded)
188 return 0;
191 * If we are respecting case, just use the original
192 * code to build the "istate->name_hash". We don't
193 * need the complexity here.
195 if (!ignore_case)
196 return 0;
198 nr_cpus = online_cpus();
199 if (nr_cpus < 2)
200 return 0;
202 if (istate->cache_nr < 2 * LAZY_THREAD_COST)
203 return 0;
205 if (istate->cache_nr < nr_cpus * LAZY_THREAD_COST)
206 nr_cpus = istate->cache_nr / LAZY_THREAD_COST;
207 lazy_nr_dir_threads = nr_cpus;
208 return lazy_nr_dir_threads;
212 * Initialize n mutexes for use when searching and inserting
213 * into "istate->dir_hash". All "dir" threads are trying
214 * to insert partial pathnames into the hash as they iterate
215 * over their portions of the index, so lock contention is
216 * high.
218 * However, the hashmap is going to put items into bucket
219 * chains based on their hash values. Use that to create n
220 * mutexes and lock on mutex[bucket(hash) % n]. This will
221 * decrease the collision rate by (hopefully) a factor of n.
223 static void init_dir_mutex(void)
225 int j;
227 lazy_dir_mutex_array = xcalloc(LAZY_MAX_MUTEX, sizeof(pthread_mutex_t));
229 for (j = 0; j < LAZY_MAX_MUTEX; j++)
230 init_recursive_mutex(&lazy_dir_mutex_array[j]);
233 static void cleanup_dir_mutex(void)
235 int j;
237 for (j = 0; j < LAZY_MAX_MUTEX; j++)
238 pthread_mutex_destroy(&lazy_dir_mutex_array[j]);
240 free(lazy_dir_mutex_array);
243 static void lock_dir_mutex(int j)
245 pthread_mutex_lock(&lazy_dir_mutex_array[j]);
248 static void unlock_dir_mutex(int j)
250 pthread_mutex_unlock(&lazy_dir_mutex_array[j]);
253 static inline int compute_dir_lock_nr(
254 const struct hashmap *map,
255 unsigned int hash)
257 return hashmap_bucket(map, hash) % LAZY_MAX_MUTEX;
260 static struct dir_entry *hash_dir_entry_with_parent_and_prefix(
261 struct index_state *istate,
262 struct dir_entry *parent,
263 struct strbuf *prefix)
265 struct dir_entry *dir;
266 unsigned int hash;
267 int lock_nr;
270 * Either we have a parent directory and path with slash(es)
271 * or the directory is an immediate child of the root directory.
273 assert((parent != NULL) ^ (strchr(prefix->buf, '/') == NULL));
275 if (parent)
276 hash = memihash_cont(parent->ent.hash,
277 prefix->buf + parent->namelen,
278 prefix->len - parent->namelen);
279 else
280 hash = memihash(prefix->buf, prefix->len);
282 lock_nr = compute_dir_lock_nr(&istate->dir_hash, hash);
283 lock_dir_mutex(lock_nr);
285 dir = find_dir_entry__hash(istate, prefix->buf, prefix->len, hash);
286 if (!dir) {
287 FLEX_ALLOC_MEM(dir, name, prefix->buf, prefix->len);
288 hashmap_entry_init(&dir->ent, hash);
289 dir->namelen = prefix->len;
290 dir->parent = parent;
291 hashmap_add(&istate->dir_hash, &dir->ent);
293 if (parent) {
294 unlock_dir_mutex(lock_nr);
296 /* All I really need here is an InterlockedIncrement(&(parent->nr)) */
297 lock_nr = compute_dir_lock_nr(&istate->dir_hash, parent->ent.hash);
298 lock_dir_mutex(lock_nr);
299 parent->nr++;
303 unlock_dir_mutex(lock_nr);
305 return dir;
309 * handle_range_1() and handle_range_dir() are derived from
310 * clear_ce_flags_1() and clear_ce_flags_dir() in unpack-trees.c
311 * and handle the iteration over the entire array of index entries.
312 * They use recursion for adjacent entries in the same parent
313 * directory.
315 static int handle_range_1(
316 struct index_state *istate,
317 int k_start,
318 int k_end,
319 struct dir_entry *parent,
320 struct strbuf *prefix,
321 struct lazy_entry *lazy_entries);
323 static int handle_range_dir(
324 struct index_state *istate,
325 int k_start,
326 int k_end,
327 struct dir_entry *parent,
328 struct strbuf *prefix,
329 struct lazy_entry *lazy_entries,
330 struct dir_entry **dir_new_out)
332 int rc, k;
333 int input_prefix_len = prefix->len;
334 struct dir_entry *dir_new;
336 dir_new = hash_dir_entry_with_parent_and_prefix(istate, parent, prefix);
338 strbuf_addch(prefix, '/');
341 * Scan forward in the index array for index entries having the same
342 * path prefix (that are also in this directory).
344 if (k_start + 1 >= k_end)
345 k = k_end;
346 else if (strncmp(istate->cache[k_start + 1]->name, prefix->buf, prefix->len) > 0)
347 k = k_start + 1;
348 else if (strncmp(istate->cache[k_end - 1]->name, prefix->buf, prefix->len) == 0)
349 k = k_end;
350 else {
351 int begin = k_start;
352 int end = k_end;
353 assert(begin >= 0);
354 while (begin < end) {
355 int mid = begin + ((end - begin) >> 1);
356 int cmp = strncmp(istate->cache[mid]->name, prefix->buf, prefix->len);
357 if (cmp == 0) /* mid has same prefix; look in second part */
358 begin = mid + 1;
359 else if (cmp > 0) /* mid is past group; look in first part */
360 end = mid;
361 else
362 die("cache entry out of order");
364 k = begin;
368 * Recurse and process what we can of this subset [k_start, k).
370 rc = handle_range_1(istate, k_start, k, dir_new, prefix, lazy_entries);
372 strbuf_setlen(prefix, input_prefix_len);
374 *dir_new_out = dir_new;
375 return rc;
378 static int handle_range_1(
379 struct index_state *istate,
380 int k_start,
381 int k_end,
382 struct dir_entry *parent,
383 struct strbuf *prefix,
384 struct lazy_entry *lazy_entries)
386 int input_prefix_len = prefix->len;
387 int k = k_start;
389 while (k < k_end) {
390 struct cache_entry *ce_k = istate->cache[k];
391 const char *name, *slash;
393 if (prefix->len && strncmp(ce_k->name, prefix->buf, prefix->len))
394 break;
396 name = ce_k->name + prefix->len;
397 slash = strchr(name, '/');
399 if (slash) {
400 int len = slash - name;
401 int processed;
402 struct dir_entry *dir_new;
404 strbuf_add(prefix, name, len);
405 processed = handle_range_dir(istate, k, k_end, parent, prefix, lazy_entries, &dir_new);
406 if (processed) {
407 k += processed;
408 strbuf_setlen(prefix, input_prefix_len);
409 continue;
412 strbuf_addch(prefix, '/');
413 processed = handle_range_1(istate, k, k_end, dir_new, prefix, lazy_entries);
414 k += processed;
415 strbuf_setlen(prefix, input_prefix_len);
416 continue;
420 * It is too expensive to take a lock to insert "ce_k"
421 * into "istate->name_hash" and increment the ref-count
422 * on the "parent" dir. So we defer actually updating
423 * permanent data structures until phase 2 (where we
424 * can change the locking requirements) and simply
425 * accumulate our current results into the lazy_entries
426 * data array).
428 * We do not need to lock the lazy_entries array because
429 * we have exclusive access to the cells in the range
430 * [k_start,k_end) that this thread was given.
432 lazy_entries[k].dir = parent;
433 if (parent) {
434 lazy_entries[k].hash_name = memihash_cont(
435 parent->ent.hash,
436 ce_k->name + parent->namelen,
437 ce_namelen(ce_k) - parent->namelen);
438 lazy_entries[k].hash_dir = parent->ent.hash;
439 } else {
440 lazy_entries[k].hash_name = memihash(ce_k->name, ce_namelen(ce_k));
443 k++;
446 return k - k_start;
449 struct lazy_dir_thread_data {
450 pthread_t pthread;
451 struct index_state *istate;
452 struct lazy_entry *lazy_entries;
453 int k_start;
454 int k_end;
457 static void *lazy_dir_thread_proc(void *_data)
459 struct lazy_dir_thread_data *d = _data;
460 struct strbuf prefix = STRBUF_INIT;
461 handle_range_1(d->istate, d->k_start, d->k_end, NULL, &prefix, d->lazy_entries);
462 strbuf_release(&prefix);
463 return NULL;
466 struct lazy_name_thread_data {
467 pthread_t pthread;
468 struct index_state *istate;
469 struct lazy_entry *lazy_entries;
472 static void *lazy_name_thread_proc(void *_data)
474 struct lazy_name_thread_data *d = _data;
475 int k;
477 for (k = 0; k < d->istate->cache_nr; k++) {
478 struct cache_entry *ce_k = d->istate->cache[k];
479 ce_k->ce_flags |= CE_HASHED;
480 hashmap_entry_init(&ce_k->ent, d->lazy_entries[k].hash_name);
481 hashmap_add(&d->istate->name_hash, &ce_k->ent);
484 return NULL;
487 static inline void lazy_update_dir_ref_counts(
488 struct index_state *istate,
489 struct lazy_entry *lazy_entries)
491 int k;
493 for (k = 0; k < istate->cache_nr; k++) {
494 if (lazy_entries[k].dir)
495 lazy_entries[k].dir->nr++;
499 static void threaded_lazy_init_name_hash(
500 struct index_state *istate)
502 int err;
503 int nr_each;
504 int k_start;
505 int t;
506 struct lazy_entry *lazy_entries;
507 struct lazy_dir_thread_data *td_dir;
508 struct lazy_name_thread_data *td_name;
510 if (!HAVE_THREADS)
511 return;
513 k_start = 0;
514 nr_each = DIV_ROUND_UP(istate->cache_nr, lazy_nr_dir_threads);
516 lazy_entries = xcalloc(istate->cache_nr, sizeof(struct lazy_entry));
517 td_dir = xcalloc(lazy_nr_dir_threads, sizeof(struct lazy_dir_thread_data));
518 td_name = xcalloc(1, sizeof(struct lazy_name_thread_data));
520 init_dir_mutex();
523 * Phase 1:
524 * Build "istate->dir_hash" using n "dir" threads (and a read-only index).
526 for (t = 0; t < lazy_nr_dir_threads; t++) {
527 struct lazy_dir_thread_data *td_dir_t = td_dir + t;
528 td_dir_t->istate = istate;
529 td_dir_t->lazy_entries = lazy_entries;
530 td_dir_t->k_start = k_start;
531 k_start += nr_each;
532 if (k_start > istate->cache_nr)
533 k_start = istate->cache_nr;
534 td_dir_t->k_end = k_start;
535 err = pthread_create(&td_dir_t->pthread, NULL, lazy_dir_thread_proc, td_dir_t);
536 if (err)
537 die(_("unable to create lazy_dir thread: %s"), strerror(err));
539 for (t = 0; t < lazy_nr_dir_threads; t++) {
540 struct lazy_dir_thread_data *td_dir_t = td_dir + t;
541 if (pthread_join(td_dir_t->pthread, NULL))
542 die("unable to join lazy_dir_thread");
546 * Phase 2:
547 * Iterate over all index entries and add them to the "istate->name_hash"
548 * using a single "name" background thread.
549 * (Testing showed it wasn't worth running more than 1 thread for this.)
551 * Meanwhile, finish updating the parent directory ref-counts for each
552 * index entry using the current thread. (This step is very fast and
553 * doesn't need threading.)
555 td_name->istate = istate;
556 td_name->lazy_entries = lazy_entries;
557 err = pthread_create(&td_name->pthread, NULL, lazy_name_thread_proc, td_name);
558 if (err)
559 die(_("unable to create lazy_name thread: %s"), strerror(err));
561 lazy_update_dir_ref_counts(istate, lazy_entries);
563 err = pthread_join(td_name->pthread, NULL);
564 if (err)
565 die(_("unable to join lazy_name thread: %s"), strerror(err));
567 cleanup_dir_mutex();
569 free(td_name);
570 free(td_dir);
571 free(lazy_entries);
574 static void lazy_init_name_hash(struct index_state *istate)
577 if (istate->name_hash_initialized)
578 return;
579 trace_performance_enter();
580 hashmap_init(&istate->name_hash, cache_entry_cmp, NULL, istate->cache_nr);
581 hashmap_init(&istate->dir_hash, dir_entry_cmp, NULL, istate->cache_nr);
583 if (lookup_lazy_params(istate)) {
585 * Disable item counting and automatic rehashing because
586 * we do per-chain (mod n) locking rather than whole hashmap
587 * locking and we need to prevent the table-size from changing
588 * and bucket items from being redistributed.
590 hashmap_disable_item_counting(&istate->dir_hash);
591 threaded_lazy_init_name_hash(istate);
592 hashmap_enable_item_counting(&istate->dir_hash);
593 } else {
594 int nr;
595 for (nr = 0; nr < istate->cache_nr; nr++)
596 hash_index_entry(istate, istate->cache[nr]);
599 istate->name_hash_initialized = 1;
600 trace_performance_leave("initialize name hash");
604 * A test routine for t/helper/ sources.
606 * Returns the number of threads used or 0 when
607 * the non-threaded code path was used.
609 * Requesting threading WILL NOT override guards
610 * in lookup_lazy_params().
612 int test_lazy_init_name_hash(struct index_state *istate, int try_threaded)
614 lazy_nr_dir_threads = 0;
615 lazy_try_threaded = try_threaded;
617 lazy_init_name_hash(istate);
619 return lazy_nr_dir_threads;
622 void add_name_hash(struct index_state *istate, struct cache_entry *ce)
624 if (istate->name_hash_initialized)
625 hash_index_entry(istate, ce);
628 void remove_name_hash(struct index_state *istate, struct cache_entry *ce)
630 if (!istate->name_hash_initialized || !(ce->ce_flags & CE_HASHED))
631 return;
632 ce->ce_flags &= ~CE_HASHED;
633 hashmap_remove(&istate->name_hash, &ce->ent, ce);
635 if (ignore_case)
636 remove_dir_entry(istate, ce);
639 static int slow_same_name(const char *name1, int len1, const char *name2, int len2)
641 if (len1 != len2)
642 return 0;
644 while (len1) {
645 unsigned char c1 = *name1++;
646 unsigned char c2 = *name2++;
647 len1--;
648 if (c1 != c2) {
649 c1 = toupper(c1);
650 c2 = toupper(c2);
651 if (c1 != c2)
652 return 0;
655 return 1;
658 static int same_name(const struct cache_entry *ce, const char *name, int namelen, int icase)
660 int len = ce_namelen(ce);
663 * Always do exact compare, even if we want a case-ignoring comparison;
664 * we do the quick exact one first, because it will be the common case.
666 if (len == namelen && !memcmp(name, ce->name, len))
667 return 1;
669 if (!icase)
670 return 0;
672 return slow_same_name(name, namelen, ce->name, len);
675 int index_dir_exists(struct index_state *istate, const char *name, int namelen)
677 struct dir_entry *dir;
679 lazy_init_name_hash(istate);
680 dir = find_dir_entry(istate, name, namelen);
681 return dir && dir->nr;
684 void adjust_dirname_case(struct index_state *istate, char *name)
686 const char *startPtr = name;
687 const char *ptr = startPtr;
689 lazy_init_name_hash(istate);
690 while (*ptr) {
691 while (*ptr && *ptr != '/')
692 ptr++;
694 if (*ptr == '/') {
695 struct dir_entry *dir;
697 dir = find_dir_entry(istate, name, ptr - name);
698 if (dir) {
699 memcpy((void *)startPtr, dir->name + (startPtr - name), ptr - startPtr);
700 startPtr = ptr + 1;
702 ptr++;
707 struct cache_entry *index_file_exists(struct index_state *istate, const char *name, int namelen, int icase)
709 struct cache_entry *ce;
710 unsigned int hash = memihash(name, namelen);
712 lazy_init_name_hash(istate);
714 ce = hashmap_get_entry_from_hash(&istate->name_hash, hash, NULL,
715 struct cache_entry, ent);
716 hashmap_for_each_entry_from(&istate->name_hash, ce, ent) {
717 if (same_name(ce, name, namelen, icase))
718 return ce;
720 return NULL;
723 void free_name_hash(struct index_state *istate)
725 if (!istate->name_hash_initialized)
726 return;
727 istate->name_hash_initialized = 0;
729 hashmap_free(&istate->name_hash);
730 hashmap_free_entries(&istate->dir_hash, struct dir_entry, ent);