#include <linux/swap.h>
#include <linux/bootmem.h>
#include <linux/fs_struct.h>
-#include <linux/hardirq.h>
#include <linux/bit_spinlock.h>
#include <linux/rculist_bl.h>
#include <linux/prefetch.h>
#include <linux/ratelimit.h>
#include <linux/list_lru.h>
-#include <linux/kasan.h>
-
#include "internal.h"
#include "mount.h"
* - i_dentry, d_u.d_alias, d_inode of aliases
* dcache_hash_bucket lock protects:
* - the dcache hash table
- * s_anon bl list spinlock protects:
- * - the s_anon list (see __d_drop)
+ * s_roots bl list spinlock protects:
+ * - the s_roots list (see __d_drop)
* dentry->d_sb->s_dentry_lru_lock protects:
* - the dcache lru lists and counters
* d_lock protects:
* dentry->d_lock
* dentry->d_sb->s_dentry_lru_lock
* dcache_hash_bucket lock
- * s_anon lock
+ * s_roots lock
*
* If there is an ancestor relationship:
* dentry->d_parent->...->d_parent->d_lock
* information, yet avoid using a prime hash-size or similar.
*/
-static unsigned int d_hash_mask __read_mostly;
static unsigned int d_hash_shift __read_mostly;
static struct hlist_bl_head *dentry_hashtable __read_mostly;
static inline struct hlist_bl_head *d_hash(unsigned int hash)
{
- return dentry_hashtable + (hash >> (32 - d_hash_shift));
+ return dentry_hashtable + (hash >> d_hash_shift);
}
#define IN_LOOKUP_SHIFT 10
unsigned long a,b,mask;
for (;;) {
- a = *(unsigned long *)cs;
+ a = read_word_at_a_time(cs);
b = load_unaligned_zeropad(ct);
if (tcount < sizeof(unsigned long))
break;
* d_drop() is used mainly for stuff that wants to invalidate a dentry for some
* reason (NFS timeouts or autofs deletes).
*
- * __d_drop requires dentry->d_lock.
+ * __d_drop requires dentry->d_lock
+ * ___d_drop doesn't mark dentry as "unhashed"
+ * (dentry->d_hash.pprev will be LIST_POISON2, not NULL).
*/
-void __d_drop(struct dentry *dentry)
+static void ___d_drop(struct dentry *dentry)
{
if (!d_unhashed(dentry)) {
struct hlist_bl_head *b;
/*
* Hashed dentries are normally on the dentry hashtable,
* with the exception of those newly allocated by
- * d_obtain_alias, which are always IS_ROOT:
+ * d_obtain_root, which are always IS_ROOT:
*/
if (unlikely(IS_ROOT(dentry)))
- b = &dentry->d_sb->s_anon;
+ b = &dentry->d_sb->s_roots;
else
b = d_hash(dentry->d_name.hash);
hlist_bl_lock(b);
__hlist_bl_del(&dentry->d_hash);
- dentry->d_hash.pprev = NULL;
hlist_bl_unlock(b);
/* After this call, in-progress rcu-walk path lookup will fail. */
write_seqcount_invalidate(&dentry->d_seq);
}
}
+
+void __d_drop(struct dentry *dentry)
+{
+ ___d_drop(dentry);
+ dentry->d_hash.pprev = NULL;
+}
EXPORT_SYMBOL(__d_drop);
void d_drop(struct dentry *dentry)
spin_unlock(&parent->d_lock);
goto again;
}
- rcu_read_unlock();
- if (parent != dentry)
+ if (parent != dentry) {
spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
- else
+ if (unlikely(dentry->d_lockref.count < 0)) {
+ spin_unlock(&parent->d_lock);
+ parent = NULL;
+ }
+ } else {
parent = NULL;
+ }
+ rcu_read_unlock();
return parent;
}
sb->s_root = NULL;
do_one_tree(dentry);
- while (!hlist_bl_empty(&sb->s_anon)) {
- dentry = dget(hlist_bl_entry(hlist_bl_first(&sb->s_anon), struct dentry, d_hash));
+ while (!hlist_bl_empty(&sb->s_roots)) {
+ dentry = dget(hlist_bl_entry(hlist_bl_first(&sb->s_roots), struct dentry, d_hash));
do_one_tree(dentry);
}
}
}
atomic_set(&p->u.count, 1);
dname = p->name;
- if (IS_ENABLED(CONFIG_DCACHE_WORD_ACCESS))
- kasan_unpoison_shadow(dname,
- round_up(name->len + 1, sizeof(unsigned long)));
} else {
dname = dentry->d_iname;
}
}
EXPORT_SYMBOL(d_alloc);
+struct dentry *d_alloc_anon(struct super_block *sb)
+{
+ return __d_alloc(sb, NULL);
+}
+EXPORT_SYMBOL(d_alloc_anon);
+
struct dentry *d_alloc_cursor(struct dentry * parent)
{
- struct dentry *dentry = __d_alloc(parent->d_sb, NULL);
+ struct dentry *dentry = d_alloc_anon(parent->d_sb);
if (dentry) {
dentry->d_flags |= DCACHE_RCUACCESS | DCACHE_DENTRY_CURSOR;
dentry->d_parent = dget(parent);
struct dentry *res = NULL;
if (root_inode) {
- res = __d_alloc(root_inode->i_sb, NULL);
+ res = d_alloc_anon(root_inode->i_sb);
if (res)
d_instantiate(res, root_inode);
else
}
EXPORT_SYMBOL(d_find_any_alias);
-static struct dentry *__d_obtain_alias(struct inode *inode, int disconnected)
+static struct dentry *__d_instantiate_anon(struct dentry *dentry,
+ struct inode *inode,
+ bool disconnected)
{
- struct dentry *tmp;
struct dentry *res;
unsigned add_flags;
- if (!inode)
- return ERR_PTR(-ESTALE);
- if (IS_ERR(inode))
- return ERR_CAST(inode);
-
- res = d_find_any_alias(inode);
- if (res)
- goto out_iput;
-
- tmp = __d_alloc(inode->i_sb, NULL);
- if (!tmp) {
- res = ERR_PTR(-ENOMEM);
- goto out_iput;
- }
-
- security_d_instantiate(tmp, inode);
+ security_d_instantiate(dentry, inode);
spin_lock(&inode->i_lock);
res = __d_find_any_alias(inode);
if (res) {
spin_unlock(&inode->i_lock);
- dput(tmp);
+ dput(dentry);
goto out_iput;
}
if (disconnected)
add_flags |= DCACHE_DISCONNECTED;
- spin_lock(&tmp->d_lock);
- __d_set_inode_and_type(tmp, inode, add_flags);
- hlist_add_head(&tmp->d_u.d_alias, &inode->i_dentry);
- hlist_bl_lock(&tmp->d_sb->s_anon);
- hlist_bl_add_head(&tmp->d_hash, &tmp->d_sb->s_anon);
- hlist_bl_unlock(&tmp->d_sb->s_anon);
- spin_unlock(&tmp->d_lock);
+ spin_lock(&dentry->d_lock);
+ __d_set_inode_and_type(dentry, inode, add_flags);
+ hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
+ if (!disconnected) {
+ hlist_bl_lock(&dentry->d_sb->s_roots);
+ hlist_bl_add_head(&dentry->d_hash, &dentry->d_sb->s_roots);
+ hlist_bl_unlock(&dentry->d_sb->s_roots);
+ }
+ spin_unlock(&dentry->d_lock);
spin_unlock(&inode->i_lock);
- return tmp;
+ return dentry;
out_iput:
iput(inode);
return res;
}
+struct dentry *d_instantiate_anon(struct dentry *dentry, struct inode *inode)
+{
+ return __d_instantiate_anon(dentry, inode, true);
+}
+EXPORT_SYMBOL(d_instantiate_anon);
+
+static struct dentry *__d_obtain_alias(struct inode *inode, bool disconnected)
+{
+ struct dentry *tmp;
+ struct dentry *res;
+
+ if (!inode)
+ return ERR_PTR(-ESTALE);
+ if (IS_ERR(inode))
+ return ERR_CAST(inode);
+
+ res = d_find_any_alias(inode);
+ if (res)
+ goto out_iput;
+
+ tmp = d_alloc_anon(inode->i_sb);
+ if (!tmp) {
+ res = ERR_PTR(-ENOMEM);
+ goto out_iput;
+ }
+
+ return __d_instantiate_anon(tmp, inode, disconnected);
+
+out_iput:
+ iput(inode);
+ return res;
+}
+
/**
* d_obtain_alias - find or allocate a DISCONNECTED dentry for a given inode
* @inode: inode to allocate the dentry for
*/
struct dentry *d_obtain_alias(struct inode *inode)
{
- return __d_obtain_alias(inode, 1);
+ return __d_obtain_alias(inode, true);
}
EXPORT_SYMBOL(d_obtain_alias);
*/
struct dentry *d_obtain_root(struct inode *inode)
{
- return __d_obtain_alias(inode, 0);
+ return __d_obtain_alias(inode, false);
}
EXPORT_SYMBOL(d_obtain_root);
static void __d_rehash(struct dentry *entry)
{
struct hlist_bl_head *b = d_hash(entry->d_name.hash);
- BUG_ON(!d_unhashed(entry));
+
hlist_bl_lock(b);
hlist_bl_add_head_rcu(&entry->d_hash, b);
hlist_bl_unlock(b);
retry:
rcu_read_lock();
- seq = smp_load_acquire(&parent->d_inode->i_dir_seq) & ~1;
+ seq = smp_load_acquire(&parent->d_inode->i_dir_seq);
r_seq = read_seqbegin(&rename_lock);
dentry = __d_lookup_rcu(parent, name, &d_seq);
if (unlikely(dentry)) {
rcu_read_unlock();
goto retry;
}
+
+ if (unlikely(seq & 1)) {
+ rcu_read_unlock();
+ goto retry;
+ }
+
hlist_bl_lock(b);
- if (unlikely(parent->d_inode->i_dir_seq != seq)) {
+ if (unlikely(READ_ONCE(parent->d_inode->i_dir_seq) != seq)) {
hlist_bl_unlock(b);
rcu_read_unlock();
goto retry;
write_seqcount_begin_nested(&target->d_seq, DENTRY_D_LOCK_NESTED);
/* unhash both */
- /* __d_drop does write_seqcount_barrier, but they're OK to nest. */
- __d_drop(dentry);
- __d_drop(target);
+ /* ___d_drop does write_seqcount_barrier, but they're OK to nest. */
+ ___d_drop(dentry);
+ ___d_drop(target);
/* Switch the names.. */
if (exchange)
__d_rehash(dentry);
if (exchange)
__d_rehash(target);
+ else
+ target->d_hash.pprev = NULL;
/* ... and switch them in the tree */
if (IS_ROOT(dentry)) {
return result;
}
+EXPORT_SYMBOL(is_subdir);
static enum d_walk_ret d_genocide_kill(void *data, struct dentry *dentry)
{
13,
HASH_EARLY | HASH_ZERO,
&d_hash_shift,
- &d_hash_mask,
+ NULL,
0,
0);
+ d_hash_shift = 32 - d_hash_shift;
}
static void __init dcache_init(void)
* but it is probably not worth it because of the cache nature
* of the dcache.
*/
- dentry_cache = KMEM_CACHE(dentry,
- SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|SLAB_MEM_SPREAD|SLAB_ACCOUNT);
+ dentry_cache = KMEM_CACHE_USERCOPY(dentry,
+ SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|SLAB_MEM_SPREAD|SLAB_ACCOUNT,
+ d_iname);
/* Hash may have been set up in dcache_init_early */
if (!hashdist)
13,
HASH_ZERO,
&d_hash_shift,
- &d_hash_mask,
+ NULL,
0,
0);
+ d_hash_shift = 32 - d_hash_shift;
}
/* SLAB cache for __getname() consumers */
void __init vfs_caches_init(void)
{
- names_cachep = kmem_cache_create("names_cache", PATH_MAX, 0,
- SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
+ names_cachep = kmem_cache_create_usercopy("names_cache", PATH_MAX, 0,
+ SLAB_HWCACHE_ALIGN|SLAB_PANIC, 0, PATH_MAX, NULL);
dcache_init();
inode_init();
git.samba.org / sfrench / cifs-2.6.git / blobdiff