From 844a391799c25d9ba85cbce33e4697db06083ec6 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 15 Feb 2011 00:38:26 -0500 Subject: [PATCH 1/5] nothing in do_follow_link() is going to see RCU Signed-off-by: Al Viro --- fs/namei.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/fs/namei.c b/fs/namei.c index ec4b2d0190a8..9ce6d272f4f2 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -668,9 +668,6 @@ force_reval_path(struct path *path, struct nameidata *nd) return 0; if (!status) { - /* Don't d_invalidate in rcu-walk mode */ - if (nameidata_drop_rcu(nd)) - return -ECHILD; d_invalidate(dentry); status = -ESTALE; } @@ -777,6 +774,8 @@ __do_follow_link(const struct path *link, struct nameidata *nd, void **p) int error; struct dentry *dentry = link->dentry; + BUG_ON(nd->flags & LOOKUP_RCU); + touch_atime(link->mnt, dentry); nd_set_link(nd, NULL); @@ -811,6 +810,11 @@ static inline int do_follow_link(struct path *path, struct nameidata *nd) { void *cookie; int err = -ELOOP; + + /* We drop rcu-walk here */ + if (nameidata_dentry_drop_rcu_maybe(nd, path->dentry)) + return -ECHILD; + if (current->link_count >= MAX_NESTED_LINKS) goto loop; if (current->total_link_count >= 40) @@ -1419,9 +1423,6 @@ static int link_path_walk(const char *name, struct nameidata *nd) goto out_dput; if (inode->i_op->follow_link) { - /* We commonly drop rcu-walk here */ - if (nameidata_dentry_drop_rcu_maybe(nd, next.dentry)) - return -ECHILD; BUG_ON(inode != next.dentry->d_inode); err = do_follow_link(&next, nd); if (err) @@ -1467,8 +1468,6 @@ static int link_path_walk(const char *name, struct nameidata *nd) break; if (inode && unlikely(inode->i_op->follow_link) && (lookup_flags & LOOKUP_FOLLOW)) { - if (nameidata_dentry_drop_rcu_maybe(nd, next.dentry)) - return -ECHILD; BUG_ON(inode != next.dentry->d_inode); err = do_follow_link(&next, nd); if (err) From 24643087e748bf192f1182766716e522dc1c972f Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 15 Feb 2011 01:26:22 -0500 Subject: [PATCH 2/5] in do_lookup() split RCU and non-RCU cases of need_revalidate and use unlikely() instead of gotos, for fsck sake... Signed-off-by: Al Viro --- fs/namei.c | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/fs/namei.c b/fs/namei.c index 9ce6d272f4f2..7609bacc7046 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -1259,9 +1259,15 @@ static int do_lookup(struct nameidata *nd, struct qstr *name, return -ECHILD; nd->seq = seq; - if (dentry->d_flags & DCACHE_OP_REVALIDATE) - goto need_revalidate; -done2: + if (unlikely(dentry->d_flags & DCACHE_OP_REVALIDATE)) { + dentry = do_revalidate(dentry, nd); + if (!dentry) + goto need_lookup; + if (IS_ERR(dentry)) + goto fail; + if (!(nd->flags & LOOKUP_RCU)) + goto done; + } path->mnt = mnt; path->dentry = dentry; if (likely(__follow_mount_rcu(nd, path, inode, false))) @@ -1274,8 +1280,13 @@ static int do_lookup(struct nameidata *nd, struct qstr *name, if (!dentry) goto need_lookup; found: - if (dentry->d_flags & DCACHE_OP_REVALIDATE) - goto need_revalidate; + if (unlikely(dentry->d_flags & DCACHE_OP_REVALIDATE)) { + dentry = do_revalidate(dentry, nd); + if (!dentry) + goto need_lookup; + if (IS_ERR(dentry)) + goto fail; + } done: path->mnt = mnt; path->dentry = dentry; @@ -1317,16 +1328,6 @@ static int do_lookup(struct nameidata *nd, struct qstr *name, mutex_unlock(&dir->i_mutex); goto found; -need_revalidate: - dentry = do_revalidate(dentry, nd); - if (!dentry) - goto need_lookup; - if (IS_ERR(dentry)) - goto fail; - if (nd->flags & LOOKUP_RCU) - goto done2; - goto done; - fail: return PTR_ERR(dentry); } From f5e1c1c1afc1d979e2ac6a24cc99ba7143639f4d Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 15 Feb 2011 01:32:55 -0500 Subject: [PATCH 3/5] split do_revalidate() into RCU and non-RCU cases fixing oopsen in lookup_one_len() Signed-off-by: Al Viro --- fs/namei.c | 47 ++++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/fs/namei.c b/fs/namei.c index 7609bacc7046..a98f7f141780 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -592,12 +592,10 @@ static int d_revalidate(struct dentry *dentry, struct nameidata *nd) return status; } -static inline struct dentry * +static struct dentry * do_revalidate(struct dentry *dentry, struct nameidata *nd) { - int status; - - status = d_revalidate(dentry, nd); + int status = d_revalidate(dentry, nd); if (unlikely(status <= 0)) { /* * The dentry failed validation. @@ -606,24 +604,39 @@ do_revalidate(struct dentry *dentry, struct nameidata *nd) * to return a fail status. */ if (status < 0) { - /* If we're in rcu-walk, we don't have a ref */ - if (!(nd->flags & LOOKUP_RCU)) - dput(dentry); + dput(dentry); dentry = ERR_PTR(status); - - } else { - /* Don't d_invalidate in rcu-walk mode */ - if (nameidata_dentry_drop_rcu_maybe(nd, dentry)) - return ERR_PTR(-ECHILD); - if (!d_invalidate(dentry)) { - dput(dentry); - dentry = NULL; - } + } else if (!d_invalidate(dentry)) { + dput(dentry); + dentry = NULL; } } return dentry; } +static inline struct dentry * +do_revalidate_rcu(struct dentry *dentry, struct nameidata *nd) +{ + int status = dentry->d_op->d_revalidate(dentry, nd); + if (likely(status > 0)) + return dentry; + if (status == -ECHILD) { + if (nameidata_dentry_drop_rcu(nd, dentry)) + return ERR_PTR(-ECHILD); + return do_revalidate(dentry, nd); + } + if (status < 0) + return ERR_PTR(status); + /* Don't d_invalidate in rcu-walk mode */ + if (nameidata_dentry_drop_rcu(nd, dentry)) + return ERR_PTR(-ECHILD); + if (!d_invalidate(dentry)) { + dput(dentry); + dentry = NULL; + } + return dentry; +} + static inline int need_reval_dot(struct dentry *dentry) { if (likely(!(dentry->d_flags & DCACHE_OP_REVALIDATE))) @@ -1260,7 +1273,7 @@ static int do_lookup(struct nameidata *nd, struct qstr *name, nd->seq = seq; if (unlikely(dentry->d_flags & DCACHE_OP_REVALIDATE)) { - dentry = do_revalidate(dentry, nd); + dentry = do_revalidate_rcu(dentry, nd); if (!dentry) goto need_lookup; if (IS_ERR(dentry)) From f60aef7ec625236a6366722bb1be7b37596bf0ae Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 15 Feb 2011 01:35:28 -0500 Subject: [PATCH 4/5] drop out of RCU in return_reval ... thus killing the need to handle drop-from-RCU in d_revalidate() Signed-off-by: Al Viro --- fs/namei.c | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/fs/namei.c b/fs/namei.c index a98f7f141780..10635d329175 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -571,25 +571,9 @@ void release_open_intent(struct nameidata *nd) } } -/* - * Call d_revalidate and handle filesystems that request rcu-walk - * to be dropped. This may be called and return in rcu-walk mode, - * regardless of success or error. If -ECHILD is returned, the caller - * must return -ECHILD back up the path walk stack so path walk may - * be restarted in ref-walk mode. - */ -static int d_revalidate(struct dentry *dentry, struct nameidata *nd) +static inline int d_revalidate(struct dentry *dentry, struct nameidata *nd) { - int status; - - status = dentry->d_op->d_revalidate(dentry, nd); - if (status == -ECHILD) { - if (nameidata_dentry_drop_rcu(nd, dentry)) - return status; - status = dentry->d_op->d_revalidate(dentry, nd); - } - - return status; + return dentry->d_op->d_revalidate(dentry, nd); } static struct dentry * @@ -617,7 +601,7 @@ do_revalidate(struct dentry *dentry, struct nameidata *nd) static inline struct dentry * do_revalidate_rcu(struct dentry *dentry, struct nameidata *nd) { - int status = dentry->d_op->d_revalidate(dentry, nd); + int status = d_revalidate(dentry, nd); if (likely(status > 0)) return dentry; if (status == -ECHILD) { @@ -1517,12 +1501,15 @@ static int link_path_walk(const char *name, struct nameidata *nd) * We may need to check the cached dentry for staleness. */ if (need_reval_dot(nd->path.dentry)) { + if (nameidata_drop_rcu_last_maybe(nd)) + return -ECHILD; /* Note: we do not d_invalidate() */ err = d_revalidate(nd->path.dentry, nd); if (!err) err = -ESTALE; if (err < 0) break; + return 0; } return_base: if (nameidata_drop_rcu_last_maybe(nd)) From 4e924a4f53a0e1ea060bd50695a12a238b250322 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 15 Feb 2011 01:42:59 -0500 Subject: [PATCH 5/5] get rid of nameidata_dentry_drop_rcu() calling nameidata_drop_rcu() can't happen anymore and didn't work right anyway Signed-off-by: Al Viro --- fs/namei.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/fs/namei.c b/fs/namei.c index 10635d329175..9e701e28a329 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -455,14 +455,6 @@ static int nameidata_dentry_drop_rcu(struct nameidata *nd, struct dentry *dentry struct fs_struct *fs = current->fs; struct dentry *parent = nd->path.dentry; - /* - * It can be possible to revalidate the dentry that we started - * the path walk with. force_reval_path may also revalidate the - * dentry already committed to the nameidata. - */ - if (unlikely(parent == dentry)) - return nameidata_drop_rcu(nd); - BUG_ON(!(nd->flags & LOOKUP_RCU)); if (nd->root.mnt) { spin_lock(&fs->lock);