5 files changed, 37 insertions, 20 deletions
diff --git a/fs/overlayfs/inode.c b/fs/overlayfs/inode.c
index 22c677040b35..4c30d44905ef 100644
--- a/fs/overlayfs/inode.c
+++ b/fs/overlayfs/inode.c
@@ -453,12 +453,12 @@ struct inode *ovl_new_inode(struct super_block *sb, umode_t mode, dev_t rdev)
 
 static int ovl_inode_test(struct inode *inode, void *data)
 {
-	return ovl_inode_real(inode, NULL) == data;
+	return inode->i_private == data;
 }
 
 static int ovl_inode_set(struct inode *inode, void *data)
 {
-	inode->i_private = (void *) (((unsigned long) data) | OVL_ISUPPER_MASK);
+	inode->i_private = data;
 	return 0;
 }
 
diff --git a/fs/overlayfs/overlayfs.h b/fs/overlayfs/overlayfs.h
index 3af33d3166e2..6e6600ae1d54 100644
--- a/fs/overlayfs/overlayfs.h
+++ b/fs/overlayfs/overlayfs.h
@@ -60,8 +60,6 @@ struct ovl_fh {
 	u8 fid[0];	/* file identifier */
 } __packed;
 
-#define OVL_ISUPPER_MASK 1UL
-
 static inline int ovl_do_rmdir(struct inode *dir, struct dentry *dentry)
 {
 	int err = vfs_rmdir(dir, dentry);
@@ -175,16 +173,6 @@ static inline struct dentry *ovl_do_tmpfile(struct dentry *dentry, umode_t mode)
 	return ret;
 }
 
-static inline struct inode *ovl_inode_real(struct inode *inode, bool *is_upper)
-{
-	unsigned long x = (unsigned long) READ_ONCE(inode->i_private);
-
-	if (is_upper)
-		*is_upper = x & OVL_ISUPPER_MASK;
-
-	return (struct inode *) (x & ~OVL_ISUPPER_MASK);
-}
-
 /* util.c */
 int ovl_want_write(struct dentry *dentry);
 void ovl_drop_write(struct dentry *dentry);
@@ -201,6 +189,7 @@ enum ovl_path_type ovl_path_real(struct dentry *dentry, struct path *path);
 struct dentry *ovl_dentry_upper(struct dentry *dentry);
 struct dentry *ovl_dentry_lower(struct dentry *dentry);
 struct dentry *ovl_dentry_real(struct dentry *dentry);
+struct inode *ovl_inode_real(struct inode *inode, bool *is_upper);
 struct ovl_dir_cache *ovl_dir_cache(struct dentry *dentry);
 void ovl_set_dir_cache(struct dentry *dentry, struct ovl_dir_cache *cache);
 bool ovl_dentry_is_opaque(struct dentry *dentry);
diff --git a/fs/overlayfs/ovl_entry.h b/fs/overlayfs/ovl_entry.h
index 553727df886c..b8c213891e84 100644
--- a/fs/overlayfs/ovl_entry.h
+++ b/fs/overlayfs/ovl_entry.h
@@ -61,6 +61,8 @@ static inline struct dentry *ovl_upperdentry_dereference(struct ovl_entry *oe)
 
 struct ovl_inode {
 	struct inode vfs_inode;
+	struct inode *upper;
+	struct inode *lower;
 };
 
 static inline struct ovl_inode *OVL_I(struct inode *inode)
diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c
index ec1b40816483..c166c1d76890 100644
--- a/fs/overlayfs/super.c
+++ b/fs/overlayfs/super.c
@@ -171,6 +171,9 @@ static struct inode *ovl_alloc_inode(struct super_block *sb)
 {
 	struct ovl_inode *oi = kmem_cache_alloc(ovl_inode_cachep, GFP_KERNEL);
 
+	oi->upper = NULL;
+	oi->lower = NULL;
+
 	return &oi->vfs_inode;
 }
 
diff --git a/fs/overlayfs/util.c b/fs/overlayfs/util.c
index f4847eff3284..fc7b1447435b 100644
--- a/fs/overlayfs/util.c
+++ b/fs/overlayfs/util.c
@@ -155,6 +155,22 @@ struct dentry *ovl_dentry_real(struct dentry *dentry)
 	return realdentry;
 }
 
+struct inode *ovl_inode_real(struct inode *inode, bool *is_upper)
+{
+	struct inode *realinode = lockless_dereference(OVL_I(inode)->upper);
+	bool isup = false;
+
+	if (!realinode)
+		realinode = OVL_I(inode)->lower;
+	else
+		isup = true;
+
+	if (is_upper)
+		*is_upper = isup;
+
+	return realinode;
+}
+
 struct ovl_dir_cache *ovl_dir_cache(struct dentry *dentry)
 {
 	struct ovl_entry *oe = dentry->d_fsdata;
@@ -233,10 +249,11 @@ void ovl_dentry_update(struct dentry *dentry, struct dentry *upperdentry)
 void ovl_inode_init(struct inode *inode, struct dentry *dentry)
 {
 	struct inode *realinode = d_inode(ovl_dentry_real(dentry));
-	bool is_upper = ovl_dentry_upper(dentry);
 
-	WRITE_ONCE(inode->i_private, (unsigned long) realinode |
-		   (is_upper ? OVL_ISUPPER_MASK : 0));
+	if (ovl_dentry_upper(dentry))
+		OVL_I(inode)->upper = realinode;
+	else
+		OVL_I(inode)->lower = realinode;
 
 	ovl_copyattr(realinode, inode);
 }
@@ -245,10 +262,16 @@ void ovl_inode_update(struct inode *inode, struct inode *upperinode)
 {
 	WARN_ON(!upperinode);
 	WARN_ON(!inode_unhashed(inode));
-	WRITE_ONCE(inode->i_private,
-		   (unsigned long) upperinode | OVL_ISUPPER_MASK);
-	if (!S_ISDIR(upperinode->i_mode))
+	/*
+	 * Make sure upperinode is consistent before making it visible to
+	 * ovl_inode_real();
+	 */
+	smp_wmb();
+	OVL_I(inode)->upper = upperinode;
+	if (!S_ISDIR(upperinode->i_mode)) {
+		inode->i_private = upperinode;
 		__insert_inode_hash(inode, (unsigned long) upperinode);
+	}
 }
 
 void ovl_dentry_version_inc(struct dentry *dentry)