4 * Complete reimplementation
5 * (C) 1997 Thomas Schoebel-Theuer,
6 * with heavy changes by Linus Torvalds
10 * Notes on the allocation strategy:
12 * The dcache is a master of the icache - whenever a dcache entry
13 * exists, the inode will always exist. "iput()" is done either when
14 * the dcache entry is deleted or garbage collected.
17 #include <linux/config.h>
18 #include <linux/string.h>
21 #include <linux/slab.h>
22 #include <linux/init.h>
23 #include <linux/smp_lock.h>
24 #include <linux/cache.h>
25 #include <linux/module.h>
26 #include <linux/mount.h>
27 #include <linux/file.h>
28 #include <asm/uaccess.h>
29 #include <linux/security.h>
30 #include <linux/seqlock.h>
32 #define DCACHE_PARANOIA 1
33 /* #define DCACHE_DEBUG 1 */
35 spinlock_t dcache_lock __cacheline_aligned_in_smp
= SPIN_LOCK_UNLOCKED
;
36 seqlock_t rename_lock __cacheline_aligned_in_smp
= SEQLOCK_UNLOCKED
;
38 static kmem_cache_t
*dentry_cache
;
41 * This is the single most critical data structure when it comes
42 * to the dcache: the hashtable for lookups. Somebody should try
43 * to make this good - I've just made it work.
45 * This hash-function tries to avoid losing too many bits of hash
46 * information, yet avoid using a prime hash-size or similar.
48 #define D_HASHBITS d_hash_shift
49 #define D_HASHMASK d_hash_mask
51 static unsigned int d_hash_mask
;
52 static unsigned int d_hash_shift
;
53 static struct hlist_head
*dentry_hashtable
;
54 static LIST_HEAD(dentry_unused
);
56 /* Statistics gathering. */
57 struct dentry_stat_t dentry_stat
= {
61 static void d_callback(void *arg
)
63 struct dentry
* dentry
= (struct dentry
*)arg
;
65 if (dname_external(dentry
)) {
66 kfree(dentry
->d_qstr
);
68 kmem_cache_free(dentry_cache
, dentry
);
72 * no dcache_lock, please. The caller must decrement dentry_stat.nr_dentry
75 static void d_free(struct dentry
*dentry
)
77 if (dentry
->d_op
&& dentry
->d_op
->d_release
)
78 dentry
->d_op
->d_release(dentry
);
79 call_rcu(&dentry
->d_rcu
, d_callback
, dentry
);
83 * Release the dentry's inode, using the filesystem
84 * d_iput() operation if defined.
85 * Called with dcache_lock held, drops it.
87 static inline void dentry_iput(struct dentry
* dentry
)
89 struct inode
*inode
= dentry
->d_inode
;
91 dentry
->d_inode
= NULL
;
92 list_del_init(&dentry
->d_alias
);
93 spin_unlock(&dcache_lock
);
94 if (dentry
->d_op
&& dentry
->d_op
->d_iput
)
95 dentry
->d_op
->d_iput(dentry
, inode
);
99 spin_unlock(&dcache_lock
);
105 * This is complicated by the fact that we do not want to put
106 * dentries that are no longer on any hash chain on the unused
107 * list: we'd much rather just get rid of them immediately.
109 * However, that implies that we have to traverse the dentry
110 * tree upwards to the parents which might _also_ now be
111 * scheduled for deletion (it may have been only waiting for
112 * its last child to go away).
114 * This tail recursion is done by hand as we don't want to depend
115 * on the compiler to always get this right (gcc generally doesn't).
116 * Real recursion would eat up our stack space.
120 * dput - release a dentry
121 * @dentry: dentry to release
123 * Release a dentry. This will drop the usage count and if appropriate
124 * call the dentry unlink method as well as removing it from the queues and
125 * releasing its resources. If the parent dentries were scheduled for release
126 * they too may now get deleted.
128 * no dcache lock, please.
131 void dput(struct dentry
*dentry
)
137 if (!atomic_dec_and_lock(&dentry
->d_count
, &dcache_lock
))
140 spin_lock(&dentry
->d_lock
);
141 if (atomic_read(&dentry
->d_count
)) {
142 spin_unlock(&dentry
->d_lock
);
143 spin_unlock(&dcache_lock
);
148 * AV: ->d_delete() is _NOT_ allowed to block now.
150 if (dentry
->d_op
&& dentry
->d_op
->d_delete
) {
151 if (dentry
->d_op
->d_delete(dentry
))
154 /* Unreachable? Get rid of it */
155 if (d_unhashed(dentry
))
157 if (list_empty(&dentry
->d_lru
)) {
158 dentry
->d_vfs_flags
|= DCACHE_REFERENCED
;
159 list_add(&dentry
->d_lru
, &dentry_unused
);
160 dentry_stat
.nr_unused
++;
162 spin_unlock(&dentry
->d_lock
);
163 spin_unlock(&dcache_lock
);
170 struct dentry
*parent
;
172 /* If dentry was on d_lru list
173 * delete it from there
175 if (!list_empty(&dentry
->d_lru
)) {
176 list_del(&dentry
->d_lru
);
177 dentry_stat
.nr_unused
--;
179 list_del(&dentry
->d_child
);
180 spin_unlock(&dentry
->d_lock
);
181 dentry_stat
.nr_dentry
--; /* For d_free, below */
182 /* drops the lock, at that point nobody can reach this dentry */
184 parent
= dentry
->d_parent
;
186 if (dentry
== parent
)
194 * d_invalidate - invalidate a dentry
195 * @dentry: dentry to invalidate
197 * Try to invalidate the dentry if it turns out to be
198 * possible. If there are other dentries that can be
199 * reached through this one we can't delete it and we
200 * return -EBUSY. On success we return 0.
205 int d_invalidate(struct dentry
* dentry
)
208 * If it's already been dropped, return OK.
210 spin_lock(&dcache_lock
);
211 if (d_unhashed(dentry
)) {
212 spin_unlock(&dcache_lock
);
216 * Check whether to do a partial shrink_dcache
217 * to get rid of unused child entries.
219 if (!list_empty(&dentry
->d_subdirs
)) {
220 spin_unlock(&dcache_lock
);
221 shrink_dcache_parent(dentry
);
222 spin_lock(&dcache_lock
);
226 * Somebody else still using it?
228 * If it's a directory, we can't drop it
229 * for fear of somebody re-populating it
230 * with children (even though dropping it
231 * would make it unreachable from the root,
232 * we might still populate it if it was a
233 * working directory or similar).
235 if (atomic_read(&dentry
->d_count
) > 1) {
236 if (dentry
->d_inode
&& S_ISDIR(dentry
->d_inode
->i_mode
)) {
237 spin_unlock(&dcache_lock
);
243 spin_unlock(&dcache_lock
);
247 /* This should be called _only_ with dcache_lock held */
249 static inline struct dentry
* __dget_locked(struct dentry
*dentry
)
251 atomic_inc(&dentry
->d_count
);
252 if (atomic_read(&dentry
->d_count
) == 1) {
253 dentry_stat
.nr_unused
--;
254 list_del_init(&dentry
->d_lru
);
259 struct dentry
* dget_locked(struct dentry
*dentry
)
261 return __dget_locked(dentry
);
265 * d_find_alias - grab a hashed alias of inode
266 * @inode: inode in question
268 * If inode has a hashed alias - acquire the reference to alias and
269 * return it. Otherwise return NULL. Notice that if inode is a directory
270 * there can be only one alias and it can be unhashed only if it has
273 * If the inode has a DCACHE_DISCONNECTED alias, then prefer
274 * any other hashed alias over that one.
277 struct dentry
* d_find_alias(struct inode
*inode
)
279 struct list_head
*head
, *next
, *tmp
;
280 struct dentry
*alias
, *discon_alias
=NULL
;
282 spin_lock(&dcache_lock
);
283 head
= &inode
->i_dentry
;
284 next
= inode
->i_dentry
.next
;
285 while (next
!= head
) {
289 alias
= list_entry(tmp
, struct dentry
, d_alias
);
290 if (!d_unhashed(alias
)) {
291 if (alias
->d_flags
& DCACHE_DISCONNECTED
)
292 discon_alias
= alias
;
294 __dget_locked(alias
);
295 spin_unlock(&dcache_lock
);
301 __dget_locked(discon_alias
);
302 spin_unlock(&dcache_lock
);
307 * Try to kill dentries associated with this inode.
308 * WARNING: you must own a reference to inode.
310 void d_prune_aliases(struct inode
*inode
)
312 struct list_head
*tmp
, *head
= &inode
->i_dentry
;
314 spin_lock(&dcache_lock
);
316 while ((tmp
= tmp
->next
) != head
) {
317 struct dentry
*dentry
= list_entry(tmp
, struct dentry
, d_alias
);
318 if (!atomic_read(&dentry
->d_count
)) {
319 __dget_locked(dentry
);
321 spin_unlock(&dcache_lock
);
326 spin_unlock(&dcache_lock
);
330 * Throw away a dentry - free the inode, dput the parent.
331 * This requires that the LRU list has already been
333 * Called with dcache_lock, drops it and then regains.
335 static inline void prune_one_dentry(struct dentry
* dentry
)
337 struct dentry
* parent
;
340 list_del(&dentry
->d_child
);
341 spin_unlock(&dentry
->d_lock
);
342 dentry_stat
.nr_dentry
--; /* For d_free, below */
344 parent
= dentry
->d_parent
;
346 if (parent
!= dentry
)
348 spin_lock(&dcache_lock
);
352 * prune_dcache - shrink the dcache
353 * @count: number of entries to try and free
355 * Shrink the dcache. This is done when we need
356 * more memory, or simply when we need to unmount
357 * something (at which point we need to unuse
360 * This function may fail to free any resources if
361 * all the dentries are in use.
364 static void prune_dcache(int count
)
366 spin_lock(&dcache_lock
);
367 for (; count
; count
--) {
368 struct dentry
*dentry
;
369 struct list_head
*tmp
;
371 tmp
= dentry_unused
.prev
;
372 if (tmp
== &dentry_unused
)
375 prefetch(dentry_unused
.prev
);
376 dentry_stat
.nr_unused
--;
377 dentry
= list_entry(tmp
, struct dentry
, d_lru
);
379 spin_lock(&dentry
->d_lock
);
380 /* leave inuse dentries */
381 if (atomic_read(&dentry
->d_count
)) {
382 spin_unlock(&dentry
->d_lock
);
385 /* If the dentry was recently referenced, don't free it. */
386 if (dentry
->d_vfs_flags
& DCACHE_REFERENCED
) {
387 dentry
->d_vfs_flags
&= ~DCACHE_REFERENCED
;
388 list_add(&dentry
->d_lru
, &dentry_unused
);
389 dentry_stat
.nr_unused
++;
390 spin_unlock(&dentry
->d_lock
);
393 prune_one_dentry(dentry
);
395 spin_unlock(&dcache_lock
);
399 * Shrink the dcache for the specified super block.
400 * This allows us to unmount a device without disturbing
401 * the dcache for the other devices.
403 * This implementation makes just two traversals of the
404 * unused list. On the first pass we move the selected
405 * dentries to the most recent end, and on the second
406 * pass we free them. The second pass must restart after
407 * each dput(), but since the target dentries are all at
408 * the end, it's really just a single traversal.
412 * shrink_dcache_sb - shrink dcache for a superblock
415 * Shrink the dcache for the specified super block. This
416 * is used to free the dcache before unmounting a file
420 void shrink_dcache_sb(struct super_block
* sb
)
422 struct list_head
*tmp
, *next
;
423 struct dentry
*dentry
;
426 * Pass one ... move the dentries for the specified
427 * superblock to the most recent end of the unused list.
429 spin_lock(&dcache_lock
);
430 next
= dentry_unused
.next
;
431 while (next
!= &dentry_unused
) {
434 dentry
= list_entry(tmp
, struct dentry
, d_lru
);
435 if (dentry
->d_sb
!= sb
)
438 list_add(tmp
, &dentry_unused
);
442 * Pass two ... free the dentries for this superblock.
445 next
= dentry_unused
.next
;
446 while (next
!= &dentry_unused
) {
449 dentry
= list_entry(tmp
, struct dentry
, d_lru
);
450 if (dentry
->d_sb
!= sb
)
452 dentry_stat
.nr_unused
--;
454 spin_lock(&dentry
->d_lock
);
455 if (atomic_read(&dentry
->d_count
)) {
456 spin_unlock(&dentry
->d_lock
);
459 prune_one_dentry(dentry
);
462 spin_unlock(&dcache_lock
);
466 * Search for at least 1 mount point in the dentry's subdirs.
467 * We descend to the next level whenever the d_subdirs
468 * list is non-empty and continue searching.
472 * have_submounts - check for mounts over a dentry
473 * @parent: dentry to check.
475 * Return true if the parent or its subdirectories contain
479 int have_submounts(struct dentry
*parent
)
481 struct dentry
*this_parent
= parent
;
482 struct list_head
*next
;
484 spin_lock(&dcache_lock
);
485 if (d_mountpoint(parent
))
488 next
= this_parent
->d_subdirs
.next
;
490 while (next
!= &this_parent
->d_subdirs
) {
491 struct list_head
*tmp
= next
;
492 struct dentry
*dentry
= list_entry(tmp
, struct dentry
, d_child
);
494 /* Have we found a mount point ? */
495 if (d_mountpoint(dentry
))
497 if (!list_empty(&dentry
->d_subdirs
)) {
498 this_parent
= dentry
;
503 * All done at this level ... ascend and resume the search.
505 if (this_parent
!= parent
) {
506 next
= this_parent
->d_child
.next
;
507 this_parent
= this_parent
->d_parent
;
510 spin_unlock(&dcache_lock
);
511 return 0; /* No mount points found in tree */
513 spin_unlock(&dcache_lock
);
518 * Search the dentry child list for the specified parent,
519 * and move any unused dentries to the end of the unused
520 * list for prune_dcache(). We descend to the next level
521 * whenever the d_subdirs list is non-empty and continue
524 static int select_parent(struct dentry
* parent
)
526 struct dentry
*this_parent
= parent
;
527 struct list_head
*next
;
530 spin_lock(&dcache_lock
);
532 next
= this_parent
->d_subdirs
.next
;
534 while (next
!= &this_parent
->d_subdirs
) {
535 struct list_head
*tmp
= next
;
536 struct dentry
*dentry
= list_entry(tmp
, struct dentry
, d_child
);
539 if (!list_empty(&dentry
->d_lru
)) {
540 dentry_stat
.nr_unused
--;
541 list_del_init(&dentry
->d_lru
);
544 * move only zero ref count dentries to the end
545 * of the unused list for prune_dcache
547 if (!atomic_read(&dentry
->d_count
)) {
548 list_add(&dentry
->d_lru
, dentry_unused
.prev
);
549 dentry_stat
.nr_unused
++;
553 * Descend a level if the d_subdirs list is non-empty.
555 if (!list_empty(&dentry
->d_subdirs
)) {
556 this_parent
= dentry
;
558 printk(KERN_DEBUG
"select_parent: descending to %s/%s, found=%d\n",
559 dentry
->d_parent
->d_name
.name
, dentry
->d_name
.name
, found
);
565 * All done at this level ... ascend and resume the search.
567 if (this_parent
!= parent
) {
568 next
= this_parent
->d_child
.next
;
569 this_parent
= this_parent
->d_parent
;
571 printk(KERN_DEBUG
"select_parent: ascending to %s/%s, found=%d\n",
572 this_parent
->d_parent
->d_name
.name
, this_parent
->d_name
.name
, found
);
576 spin_unlock(&dcache_lock
);
581 * shrink_dcache_parent - prune dcache
582 * @parent: parent of entries to prune
584 * Prune the dcache to remove unused children of the parent dentry.
587 void shrink_dcache_parent(struct dentry
* parent
)
591 while ((found
= select_parent(parent
)) != 0)
596 * shrink_dcache_anon - further prune the cache
597 * @head: head of d_hash list of dentries to prune
599 * Prune the dentries that are anonymous
601 * parsing d_hash list does not read_barrier_depends() as it
602 * done under dcache_lock.
605 void shrink_dcache_anon(struct hlist_head
*head
)
607 struct hlist_node
*lp
;
611 spin_lock(&dcache_lock
);
612 hlist_for_each(lp
, head
) {
613 struct dentry
*this = hlist_entry(lp
, struct dentry
, d_hash
);
614 if (!list_empty(&this->d_lru
)) {
615 dentry_stat
.nr_unused
--;
616 list_del(&this->d_lru
);
620 * move only zero ref count dentries to the end
621 * of the unused list for prune_dcache
623 if (!atomic_read(&this->d_count
)) {
624 list_add_tail(&this->d_lru
, &dentry_unused
);
625 dentry_stat
.nr_unused
++;
629 spin_unlock(&dcache_lock
);
635 * This is called from kswapd when we think we need some more memory.
637 * We don't want the VM to steal _all_ unused dcache. Because that leads to
638 * the VM stealing all unused inodes, which shoots down recently-used
639 * pagecache. So what we do is to tell fibs to the VM about how many reapable
640 * objects there are in this cache. If the number of unused dentries is
641 * less than half of the total dentry count then return zero. The net effect
642 * is that the number of unused dentries will be, at a minimum, equal to the
643 * number of used ones.
645 * If unused_ratio is set to 5, the number of unused dentries will not fall
646 * below 5* the number of used ones.
648 static int shrink_dcache_memory(int nr
, unsigned int gfp_mask
)
652 const int unused_ratio
= 1;
656 * Nasty deadlock avoidance.
658 * ext2_new_block->getblk->GFP->shrink_dcache_memory->
659 * prune_dcache->prune_one_dentry->dput->dentry_iput->iput->
660 * inode->i_sb->s_op->put_inode->ext2_discard_prealloc->
661 * ext2_free_blocks->lock_super->DEADLOCK.
663 * We should make sure we don't hold the superblock lock over
664 * block allocations, but for now:
666 if (gfp_mask
& __GFP_FS
)
669 nr_unused
= dentry_stat
.nr_unused
;
670 nr_used
= dentry_stat
.nr_dentry
- nr_unused
;
671 if (nr_unused
< nr_used
* unused_ratio
)
673 return nr_unused
- nr_used
* unused_ratio
;
676 #define NAME_ALLOC_LEN(len) ((len+16) & ~15)
679 * d_alloc - allocate a dcache entry
680 * @parent: parent of entry to allocate
681 * @name: qstr of the name
683 * Allocates a dentry. It returns %NULL if there is insufficient memory
684 * available. On a success the dentry is returned. The name passed in is
685 * copied and the copy passed in may be reused after this call.
688 struct dentry
* d_alloc(struct dentry
* parent
, const struct qstr
*name
)
691 struct dentry
*dentry
;
694 dentry
= kmem_cache_alloc(dentry_cache
, GFP_KERNEL
);
698 if (name
->len
> DNAME_INLINE_LEN
-1) {
699 qstr
= kmalloc(sizeof(*qstr
) + NAME_ALLOC_LEN(name
->len
),
702 kmem_cache_free(dentry_cache
, dentry
);
705 qstr
->name
= qstr
->name_str
;
706 qstr
->len
= name
->len
;
707 qstr
->hash
= name
->hash
;
708 dentry
->d_qstr
= qstr
;
709 str
= qstr
->name_str
;
711 dentry
->d_qstr
= &dentry
->d_name
;
712 str
= dentry
->d_iname
;
715 memcpy(str
, name
->name
, name
->len
);
718 atomic_set(&dentry
->d_count
, 1);
719 dentry
->d_vfs_flags
= DCACHE_UNHASHED
;
720 dentry
->d_lock
= SPIN_LOCK_UNLOCKED
;
722 dentry
->d_inode
= NULL
;
723 dentry
->d_parent
= NULL
;
724 dentry
->d_move_count
= 0;
726 dentry
->d_name
.name
= str
;
727 dentry
->d_name
.len
= name
->len
;
728 dentry
->d_name
.hash
= name
->hash
;
730 dentry
->d_fsdata
= NULL
;
731 dentry
->d_mounted
= 0;
732 dentry
->d_cookie
= NULL
;
733 dentry
->d_bucket
= NULL
;
734 INIT_HLIST_NODE(&dentry
->d_hash
);
735 INIT_LIST_HEAD(&dentry
->d_lru
);
736 INIT_LIST_HEAD(&dentry
->d_subdirs
);
737 INIT_LIST_HEAD(&dentry
->d_alias
);
740 dentry
->d_parent
= dget(parent
);
741 dentry
->d_sb
= parent
->d_sb
;
743 INIT_LIST_HEAD(&dentry
->d_child
);
746 spin_lock(&dcache_lock
);
748 list_add(&dentry
->d_child
, &parent
->d_subdirs
);
749 dentry_stat
.nr_dentry
++;
750 spin_unlock(&dcache_lock
);
756 * d_instantiate - fill in inode information for a dentry
757 * @entry: dentry to complete
758 * @inode: inode to attach to this dentry
760 * Fill in inode information in the entry.
762 * This turns negative dentries into productive full members
765 * NOTE! This assumes that the inode count has been incremented
766 * (or otherwise set) by the caller to indicate that it is now
767 * in use by the dcache.
770 void d_instantiate(struct dentry
*entry
, struct inode
* inode
)
772 if (!list_empty(&entry
->d_alias
)) BUG();
773 spin_lock(&dcache_lock
);
775 list_add(&entry
->d_alias
, &inode
->i_dentry
);
776 entry
->d_inode
= inode
;
777 spin_unlock(&dcache_lock
);
778 security_d_instantiate(entry
, inode
);
782 * d_alloc_root - allocate root dentry
783 * @root_inode: inode to allocate the root for
785 * Allocate a root ("/") dentry for the inode given. The inode is
786 * instantiated and returned. %NULL is returned if there is insufficient
787 * memory or the inode passed is %NULL.
790 struct dentry
* d_alloc_root(struct inode
* root_inode
)
792 struct dentry
*res
= NULL
;
795 static const struct qstr name
= { .name
= "/", .len
= 1, .hash
= 0 };
796 res
= d_alloc(NULL
, &name
);
798 res
->d_sb
= root_inode
->i_sb
;
800 d_instantiate(res
, root_inode
);
806 static inline struct hlist_head
* d_hash(struct dentry
* parent
, unsigned long hash
)
808 hash
+= (unsigned long) parent
/ L1_CACHE_BYTES
;
809 hash
= hash
^ (hash
>> D_HASHBITS
);
810 return dentry_hashtable
+ (hash
& D_HASHMASK
);
814 * d_alloc_anon - allocate an anonymous dentry
815 * @inode: inode to allocate the dentry for
817 * This is similar to d_alloc_root. It is used by filesystems when
818 * creating a dentry for a given inode, often in the process of
819 * mapping a filehandle to a dentry. The returned dentry may be
820 * anonymous, or may have a full name (if the inode was already
821 * in the cache). The file system may need to make further
822 * efforts to connect this dentry into the dcache properly.
824 * When called on a directory inode, we must ensure that
825 * the inode only ever has one dentry. If a dentry is
826 * found, that is returned instead of allocating a new one.
828 * On successful return, the reference to the inode has been transferred
829 * to the dentry. If %NULL is returned (indicating kmalloc failure),
830 * the reference on the inode has not been released.
833 struct dentry
* d_alloc_anon(struct inode
*inode
)
835 static const struct qstr anonstring
= { "", 0, 0};
839 if ((res
= d_find_alias(inode
))) {
844 tmp
= d_alloc(NULL
, &anonstring
);
848 tmp
->d_parent
= tmp
; /* make sure dput doesn't croak */
850 spin_lock(&dcache_lock
);
851 if (S_ISDIR(inode
->i_mode
) && !list_empty(&inode
->i_dentry
)) {
852 /* A directory can only have one dentry.
853 * This (now) has one, so use it.
855 res
= list_entry(inode
->i_dentry
.next
, struct dentry
, d_alias
);
858 /* attach a disconnected dentry */
862 spin_lock(&res
->d_lock
);
863 res
->d_sb
= inode
->i_sb
;
865 res
->d_inode
= inode
;
866 res
->d_bucket
= d_hash(res
, res
->d_name
.hash
);
867 res
->d_flags
|= DCACHE_DISCONNECTED
;
868 res
->d_vfs_flags
&= ~DCACHE_UNHASHED
;
869 list_add(&res
->d_alias
, &inode
->i_dentry
);
870 hlist_add_head(&res
->d_hash
, &inode
->i_sb
->s_anon
);
871 spin_unlock(&res
->d_lock
);
873 inode
= NULL
; /* don't drop reference */
875 spin_unlock(&dcache_lock
);
886 * d_splice_alias - splice a disconnected dentry into the tree if one exists
887 * @inode: the inode which may have a disconnected dentry
888 * @dentry: a negative dentry which we want to point to the inode.
890 * If inode is a directory and has a 'disconnected' dentry (i.e. IS_ROOT and
891 * DCACHE_DISCONNECTED), then d_move that in place of the given dentry
892 * and return it, else simply d_add the inode to the dentry and return NULL.
894 * This is (will be) needed in the lookup routine of any filesystem that is exportable
895 * (via knfsd) so that we can build dcache paths to directories effectively.
897 * If a dentry was found and moved, then it is returned. Otherwise NULL
898 * is returned. This matches the expected return value of ->lookup.
901 struct dentry
*d_splice_alias(struct inode
*inode
, struct dentry
*dentry
)
903 struct dentry
*new = NULL
;
905 if (inode
&& S_ISDIR(inode
->i_mode
)) {
906 spin_lock(&dcache_lock
);
907 if (!list_empty(&inode
->i_dentry
)) {
908 new = list_entry(inode
->i_dentry
.next
, struct dentry
, d_alias
);
910 spin_unlock(&dcache_lock
);
911 security_d_instantiate(dentry
, inode
);
916 /* d_instantiate takes dcache_lock, so we do it by hand */
917 list_add(&dentry
->d_alias
, &inode
->i_dentry
);
918 dentry
->d_inode
= inode
;
919 spin_unlock(&dcache_lock
);
920 security_d_instantiate(dentry
, inode
);
924 d_add(dentry
, inode
);
930 * d_lookup - search for a dentry
931 * @parent: parent dentry
932 * @name: qstr of name we wish to find
934 * Searches the children of the parent dentry for the name in question. If
935 * the dentry is found its reference count is incremented and the dentry
936 * is returned. The caller must use d_put to free the entry when it has
937 * finished using it. %NULL is returned on failure.
939 * __d_lookup is dcache_lock free. The hash list is protected using RCU.
940 * Memory barriers are used while updating and doing lockless traversal.
941 * To avoid races with d_move while rename is happening, d_move_count is
944 * Overflows in memcmp(), while d_move, are avoided by keeping the length
945 * and name pointer in one structure pointed by d_qstr.
947 * rcu_read_lock() and rcu_read_unlock() are used to disable preemption while
948 * lookup is going on.
950 * d_lru list is not updated, which can leave non-zero d_count dentries
951 * around in d_lru list.
953 * d_lookup() is protected against the concurrent renames in some unrelated
954 * directory using the seqlockt_t rename_lock.
957 struct dentry
* d_lookup(struct dentry
* parent
, struct qstr
* name
)
959 struct dentry
* dentry
= NULL
;
963 seq
= read_seqbegin(&rename_lock
);
964 dentry
= __d_lookup(parent
, name
);
967 } while (read_seqretry(&rename_lock
, seq
));
971 struct dentry
* __d_lookup(struct dentry
* parent
, struct qstr
* name
)
973 unsigned int len
= name
->len
;
974 unsigned int hash
= name
->hash
;
975 const unsigned char *str
= name
->name
;
976 struct hlist_head
*head
= d_hash(parent
,hash
);
977 struct dentry
*found
= NULL
;
978 struct hlist_node
*node
;
982 hlist_for_each (node
, head
) {
983 struct dentry
*dentry
;
984 unsigned long move_count
;
987 smp_read_barrier_depends();
988 dentry
= hlist_entry(node
, struct dentry
, d_hash
);
990 /* if lookup ends up in a different bucket
991 * due to concurrent rename, fail it
993 if (unlikely(dentry
->d_bucket
!= head
))
997 * We must take a snapshot of d_move_count followed by
998 * read memory barrier before any search key comparison
1000 move_count
= dentry
->d_move_count
;
1003 if (dentry
->d_name
.hash
!= hash
)
1005 if (dentry
->d_parent
!= parent
)
1008 qstr
= dentry
->d_qstr
;
1009 smp_read_barrier_depends();
1010 if (parent
->d_op
&& parent
->d_op
->d_compare
) {
1011 if (parent
->d_op
->d_compare(parent
, qstr
, name
))
1014 if (qstr
->len
!= len
)
1016 if (memcmp(qstr
->name
, str
, len
))
1019 spin_lock(&dentry
->d_lock
);
1021 * If dentry is moved, fail the lookup
1023 if (likely(move_count
== dentry
->d_move_count
)) {
1024 if (!d_unhashed(dentry
)) {
1025 atomic_inc(&dentry
->d_count
);
1029 spin_unlock(&dentry
->d_lock
);
1038 * d_validate - verify dentry provided from insecure source
1039 * @dentry: The dentry alleged to be valid child of @dparent
1040 * @dparent: The parent dentry (known to be valid)
1041 * @hash: Hash of the dentry
1042 * @len: Length of the name
1044 * An insecure source has sent us a dentry, here we verify it and dget() it.
1045 * This is used by ncpfs in its readdir implementation.
1046 * Zero is returned in the dentry is invalid.
1049 int d_validate(struct dentry
*dentry
, struct dentry
*dparent
)
1051 unsigned long dent_addr
= (unsigned long) dentry
;
1052 unsigned long min_addr
= PAGE_OFFSET
;
1053 unsigned long align_mask
= 0x0F;
1054 struct hlist_head
*base
;
1055 struct hlist_node
*lhp
;
1057 if (dent_addr
< min_addr
)
1059 if (dent_addr
> (unsigned long)high_memory
- sizeof(struct dentry
))
1061 if (dent_addr
& align_mask
)
1063 if ((!kern_addr_valid(dent_addr
)) || (!kern_addr_valid(dent_addr
-1 +
1064 sizeof(struct dentry
))))
1067 if (dentry
->d_parent
!= dparent
)
1070 spin_lock(&dcache_lock
);
1071 base
= d_hash(dparent
, dentry
->d_name
.hash
);
1072 hlist_for_each(lhp
,base
) {
1073 /* read_barrier_depends() not required for d_hash list
1074 * as it is parsed under dcache_lock
1076 if (dentry
== hlist_entry(lhp
, struct dentry
, d_hash
)) {
1077 __dget_locked(dentry
);
1078 spin_unlock(&dcache_lock
);
1082 spin_unlock(&dcache_lock
);
1088 * When a file is deleted, we have two options:
1089 * - turn this dentry into a negative dentry
1090 * - unhash this dentry and free it.
1092 * Usually, we want to just turn this into
1093 * a negative dentry, but if anybody else is
1094 * currently using the dentry or the inode
1095 * we can't do that and we fall back on removing
1096 * it from the hash queues and waiting for
1097 * it to be deleted later when it has no users
1101 * d_delete - delete a dentry
1102 * @dentry: The dentry to delete
1104 * Turn the dentry into a negative dentry if possible, otherwise
1105 * remove it from the hash queues so it can be deleted later
1108 void d_delete(struct dentry
* dentry
)
1111 * Are we the only user?
1113 spin_lock(&dcache_lock
);
1114 spin_lock(&dentry
->d_lock
);
1115 if (atomic_read(&dentry
->d_count
) == 1) {
1116 spin_unlock(&dentry
->d_lock
);
1117 dentry_iput(dentry
);
1121 if (!d_unhashed(dentry
))
1124 spin_unlock(&dentry
->d_lock
);
1125 spin_unlock(&dcache_lock
);
1129 * d_rehash - add an entry back to the hash
1130 * @entry: dentry to add to the hash
1132 * Adds a dentry to the hash according to its name.
1135 void d_rehash(struct dentry
* entry
)
1137 struct hlist_head
*list
= d_hash(entry
->d_parent
, entry
->d_name
.hash
);
1138 spin_lock(&dcache_lock
);
1139 entry
->d_vfs_flags
&= ~DCACHE_UNHASHED
;
1140 entry
->d_bucket
= list
;
1141 hlist_add_head_rcu(&entry
->d_hash
, list
);
1142 spin_unlock(&dcache_lock
);
1145 #define do_switch(x,y) do { \
1146 __typeof__ (x) __tmp = x; \
1147 x = y; y = __tmp; } while (0)
1150 * When switching names, the actual string doesn't strictly have to
1151 * be preserved in the target - because we're dropping the target
1152 * anyway. As such, we can just do a simple memcpy() to copy over
1153 * the new name before we switch.
1155 * Note that we have to be a lot more careful about getting the hash
1156 * switched - we have to switch the hash value properly even if it
1157 * then no longer matches the actual (corrupted) string of the target.
1158 * The hash value has to match the hash queue that the dentry is on..
1160 static inline void switch_names(struct dentry
* dentry
, struct dentry
* target
)
1162 const unsigned char *old_name
, *new_name
;
1163 struct qstr
*old_qstr
, *new_qstr
;
1165 memcpy(dentry
->d_iname
, target
->d_iname
, DNAME_INLINE_LEN
);
1166 old_qstr
= target
->d_qstr
;
1167 old_name
= target
->d_name
.name
;
1168 new_qstr
= dentry
->d_qstr
;
1169 new_name
= dentry
->d_name
.name
;
1170 if (old_name
== target
->d_iname
) {
1171 old_name
= dentry
->d_iname
;
1172 old_qstr
= &dentry
->d_name
;
1174 if (new_name
== dentry
->d_iname
) {
1175 new_name
= target
->d_iname
;
1176 new_qstr
= &target
->d_name
;
1178 target
->d_name
.name
= new_name
;
1179 dentry
->d_name
.name
= old_name
;
1180 target
->d_qstr
= new_qstr
;
1181 dentry
->d_qstr
= old_qstr
;
1185 * We cannibalize "target" when moving dentry on top of it,
1186 * because it's going to be thrown away anyway. We could be more
1187 * polite about it, though.
1189 * This forceful removal will result in ugly /proc output if
1190 * somebody holds a file open that got deleted due to a rename.
1191 * We could be nicer about the deleted file, and let it show
1192 * up under the name it got deleted rather than the name that
1197 * d_move - move a dentry
1198 * @dentry: entry to move
1199 * @target: new dentry
1201 * Update the dcache to reflect the move of a file name. Negative
1202 * dcache entries should not be moved in this way.
1205 void d_move(struct dentry
* dentry
, struct dentry
* target
)
1207 if (!dentry
->d_inode
)
1208 printk(KERN_WARNING
"VFS: moving negative dcache entry\n");
1210 spin_lock(&dcache_lock
);
1211 write_seqlock(&rename_lock
);
1213 * XXXX: do we really need to take target->d_lock?
1215 if (target
< dentry
) {
1216 spin_lock(&target
->d_lock
);
1217 spin_lock(&dentry
->d_lock
);
1219 spin_lock(&dentry
->d_lock
);
1220 spin_lock(&target
->d_lock
);
1223 /* Move the dentry to the target hash queue, if on different bucket */
1224 if (dentry
->d_vfs_flags
& DCACHE_UNHASHED
)
1225 goto already_unhashed
;
1226 if (dentry
->d_bucket
!= target
->d_bucket
) {
1227 hlist_del_rcu(&dentry
->d_hash
);
1229 dentry
->d_bucket
= target
->d_bucket
;
1230 hlist_add_head_rcu(&dentry
->d_hash
, target
->d_bucket
);
1231 dentry
->d_vfs_flags
&= ~DCACHE_UNHASHED
;
1234 /* Unhash the target: dput() will then get rid of it */
1237 list_del(&dentry
->d_child
);
1238 list_del(&target
->d_child
);
1240 /* Switch the names.. */
1241 switch_names(dentry
, target
);
1243 do_switch(dentry
->d_name
.len
, target
->d_name
.len
);
1244 do_switch(dentry
->d_name
.hash
, target
->d_name
.hash
);
1246 /* ... and switch the parents */
1247 if (IS_ROOT(dentry
)) {
1248 dentry
->d_parent
= target
->d_parent
;
1249 target
->d_parent
= target
;
1250 INIT_LIST_HEAD(&target
->d_child
);
1252 do_switch(dentry
->d_parent
, target
->d_parent
);
1254 /* And add them back to the (new) parent lists */
1255 list_add(&target
->d_child
, &target
->d_parent
->d_subdirs
);
1258 list_add(&dentry
->d_child
, &dentry
->d_parent
->d_subdirs
);
1259 dentry
->d_move_count
++;
1260 spin_unlock(&target
->d_lock
);
1261 spin_unlock(&dentry
->d_lock
);
1262 write_sequnlock(&rename_lock
);
1263 spin_unlock(&dcache_lock
);
1267 * d_path - return the path of a dentry
1268 * @dentry: dentry to report
1269 * @vfsmnt: vfsmnt to which the dentry belongs
1270 * @root: root dentry
1271 * @rootmnt: vfsmnt to which the root dentry belongs
1272 * @buffer: buffer to return value in
1273 * @buflen: buffer length
1275 * Convert a dentry into an ASCII path name. If the entry has been deleted
1276 * the string " (deleted)" is appended. Note that this is ambiguous.
1278 * Returns the buffer or an error code if the path was too long.
1280 * "buflen" should be positive. Caller holds the dcache_lock.
1282 static char * __d_path( struct dentry
*dentry
, struct vfsmount
*vfsmnt
,
1283 struct dentry
*root
, struct vfsmount
*rootmnt
,
1284 char *buffer
, int buflen
)
1286 char * end
= buffer
+buflen
;
1292 if (!IS_ROOT(dentry
) && d_unhashed(dentry
)) {
1297 memcpy(end
, " (deleted)", 10);
1307 struct dentry
* parent
;
1309 if (dentry
== root
&& vfsmnt
== rootmnt
)
1311 if (dentry
== vfsmnt
->mnt_root
|| IS_ROOT(dentry
)) {
1313 if (vfsmnt
->mnt_parent
== vfsmnt
)
1315 dentry
= vfsmnt
->mnt_mountpoint
;
1316 vfsmnt
= vfsmnt
->mnt_parent
;
1319 parent
= dentry
->d_parent
;
1321 namelen
= dentry
->d_name
.len
;
1322 buflen
-= namelen
+ 1;
1326 memcpy(end
, dentry
->d_name
.name
, namelen
);
1335 namelen
= dentry
->d_name
.len
;
1339 retval
-= namelen
-1; /* hit the slash */
1340 memcpy(retval
, dentry
->d_name
.name
, namelen
);
1343 return ERR_PTR(-ENAMETOOLONG
);
1346 /* write full pathname into buffer and return start of pathname */
1347 char * d_path(struct dentry
*dentry
, struct vfsmount
*vfsmnt
,
1348 char *buf
, int buflen
)
1351 struct vfsmount
*rootmnt
;
1352 struct dentry
*root
;
1353 read_lock(¤t
->fs
->lock
);
1354 rootmnt
= mntget(current
->fs
->rootmnt
);
1355 root
= dget(current
->fs
->root
);
1356 read_unlock(¤t
->fs
->lock
);
1357 spin_lock(&dcache_lock
);
1358 res
= __d_path(dentry
, vfsmnt
, root
, rootmnt
, buf
, buflen
);
1359 spin_unlock(&dcache_lock
);
1366 * NOTE! The user-level library version returns a
1367 * character pointer. The kernel system call just
1368 * returns the length of the buffer filled (which
1369 * includes the ending '\0' character), or a negative
1370 * error value. So libc would do something like
1372 * char *getcwd(char * buf, size_t size)
1376 * retval = sys_getcwd(buf, size);
1383 asmlinkage
long sys_getcwd(char __user
*buf
, unsigned long size
)
1386 struct vfsmount
*pwdmnt
, *rootmnt
;
1387 struct dentry
*pwd
, *root
;
1388 char *page
= (char *) __get_free_page(GFP_USER
);
1393 read_lock(¤t
->fs
->lock
);
1394 pwdmnt
= mntget(current
->fs
->pwdmnt
);
1395 pwd
= dget(current
->fs
->pwd
);
1396 rootmnt
= mntget(current
->fs
->rootmnt
);
1397 root
= dget(current
->fs
->root
);
1398 read_unlock(¤t
->fs
->lock
);
1401 /* Has the current directory has been unlinked? */
1402 spin_lock(&dcache_lock
);
1403 if (pwd
->d_parent
== pwd
|| !d_unhashed(pwd
)) {
1407 cwd
= __d_path(pwd
, pwdmnt
, root
, rootmnt
, page
, PAGE_SIZE
);
1408 spin_unlock(&dcache_lock
);
1410 error
= PTR_ERR(cwd
);
1415 len
= PAGE_SIZE
+ page
- cwd
;
1418 if (copy_to_user(buf
, cwd
, len
))
1422 spin_unlock(&dcache_lock
);
1429 free_page((unsigned long) page
);
1434 * Test whether new_dentry is a subdirectory of old_dentry.
1436 * Trivially implemented using the dcache structure
1440 * is_subdir - is new dentry a subdirectory of old_dentry
1441 * @new_dentry: new dentry
1442 * @old_dentry: old dentry
1444 * Returns 1 if new_dentry is a subdirectory of the parent (at any depth).
1445 * Returns 0 otherwise.
1448 int is_subdir(struct dentry
* new_dentry
, struct dentry
* old_dentry
)
1454 if (new_dentry
!= old_dentry
) {
1455 struct dentry
* parent
= new_dentry
->d_parent
;
1456 if (parent
== new_dentry
)
1458 new_dentry
= parent
;
1467 void d_genocide(struct dentry
*root
)
1469 struct dentry
*this_parent
= root
;
1470 struct list_head
*next
;
1472 spin_lock(&dcache_lock
);
1474 next
= this_parent
->d_subdirs
.next
;
1476 while (next
!= &this_parent
->d_subdirs
) {
1477 struct list_head
*tmp
= next
;
1478 struct dentry
*dentry
= list_entry(tmp
, struct dentry
, d_child
);
1480 if (d_unhashed(dentry
)||!dentry
->d_inode
)
1482 if (!list_empty(&dentry
->d_subdirs
)) {
1483 this_parent
= dentry
;
1486 atomic_dec(&dentry
->d_count
);
1488 if (this_parent
!= root
) {
1489 next
= this_parent
->d_child
.next
;
1490 atomic_dec(&this_parent
->d_count
);
1491 this_parent
= this_parent
->d_parent
;
1494 spin_unlock(&dcache_lock
);
1498 * find_inode_number - check for dentry with name
1499 * @dir: directory to check
1500 * @name: Name to find.
1502 * Check whether a dentry already exists for the given name,
1503 * and return the inode number if it has an inode. Otherwise
1506 * This routine is used to post-process directory listings for
1507 * filesystems using synthetic inode numbers, and is necessary
1508 * to keep getcwd() working.
1511 ino_t
find_inode_number(struct dentry
*dir
, struct qstr
*name
)
1513 struct dentry
* dentry
;
1517 * Check for a fs-specific hash function. Note that we must
1518 * calculate the standard hash first, as the d_op->d_hash()
1519 * routine may choose to leave the hash value unchanged.
1521 name
->hash
= full_name_hash(name
->name
, name
->len
);
1522 if (dir
->d_op
&& dir
->d_op
->d_hash
)
1524 if (dir
->d_op
->d_hash(dir
, name
) != 0)
1528 dentry
= d_lookup(dir
, name
);
1531 if (dentry
->d_inode
)
1532 ino
= dentry
->d_inode
->i_ino
;
1539 static void __init
dcache_init(unsigned long mempages
)
1541 struct hlist_head
*d
;
1542 unsigned long order
;
1543 unsigned int nr_hash
;
1547 * A constructor could be added for stable state like the lists,
1548 * but it is probably not worth it because of the cache nature
1550 * If fragmentation is too bad then the SLAB_HWCACHE_ALIGN
1551 * flag could be removed here, to hint to the allocator that
1552 * it should not try to get multiple page regions.
1554 dentry_cache
= kmem_cache_create("dentry_cache",
1555 sizeof(struct dentry
),
1557 SLAB_HWCACHE_ALIGN
|SLAB_RECLAIM_ACCOUNT
,
1560 panic("Cannot create dentry cache");
1562 set_shrinker(DEFAULT_SEEKS
, shrink_dcache_memory
);
1565 mempages
>>= (13 - PAGE_SHIFT
);
1567 mempages
*= sizeof(struct hlist_head
);
1568 for (order
= 0; ((1UL << order
) << PAGE_SHIFT
) < mempages
; order
++)
1574 nr_hash
= (1UL << order
) * PAGE_SIZE
/
1575 sizeof(struct hlist_head
);
1576 d_hash_mask
= (nr_hash
- 1);
1580 while ((tmp
>>= 1UL) != 0UL)
1583 dentry_hashtable
= (struct hlist_head
*)
1584 __get_free_pages(GFP_ATOMIC
, order
);
1585 } while (dentry_hashtable
== NULL
&& --order
>= 0);
1587 printk(KERN_INFO
"Dentry cache hash table entries: %d (order: %ld, %ld bytes)\n",
1588 nr_hash
, order
, (PAGE_SIZE
<< order
));
1590 if (!dentry_hashtable
)
1591 panic("Failed to allocate dcache hash table\n");
1593 d
= dentry_hashtable
;
1602 /* SLAB cache for __getname() consumers */
1603 kmem_cache_t
*names_cachep
;
1605 /* SLAB cache for file structures */
1606 kmem_cache_t
*filp_cachep
;
1608 EXPORT_SYMBOL(d_genocide
);
1610 extern void bdev_cache_init(void);
1611 extern void chrdev_init(void);
1613 void __init
vfs_caches_init(unsigned long mempages
)
1615 names_cachep
= kmem_cache_create("names_cache",
1617 SLAB_HWCACHE_ALIGN
, NULL
, NULL
);
1619 panic("Cannot create names SLAB cache");
1621 filp_cachep
= kmem_cache_create("filp",
1622 sizeof(struct file
), 0,
1623 SLAB_HWCACHE_ALIGN
, filp_ctor
, filp_dtor
);
1625 panic("Cannot create filp SLAB cache");
1627 dcache_init(mempages
);
1628 inode_init(mempages
);
1629 files_init(mempages
);