summaryrefslogtreecommitdiff
path: root/fs
diff options
context:
space:
mode:
authorKent Overstreet <kent.overstreet@gmail.com>2021-12-26 03:55:34 +0300
committerKent Overstreet <kent.overstreet@linux.dev>2023-10-23 00:09:20 +0300
commita786087744fcff140ecce0e1dd93a43186edf8ad (patch)
tree7d1dcd71ac2883793415da6ebe045f999ceaeb3b /fs
parent47ac34ec988f01e1e0d00a5281abe0812bad4fcc (diff)
downloadlinux-a786087744fcff140ecce0e1dd93a43186edf8ad.tar.xz
bcachefs: New in-memory array for bucket gens
The main in-memory bucket array is going away, but we'll still need to keep bucket generations in memory, at least for now - ptr_stale() needs to be an efficient operation. Signed-off-by: Kent Overstreet <kent.overstreet@gmail.com>
Diffstat (limited to 'fs')
-rw-r--r--fs/bcachefs/alloc_background.c2
-rw-r--r--fs/bcachefs/alloc_foreground.c4
-rw-r--r--fs/bcachefs/bcachefs.h1
-rw-r--r--fs/bcachefs/buckets.c43
-rw-r--r--fs/bcachefs/buckets.h20
-rw-r--r--fs/bcachefs/buckets_types.h7
6 files changed, 72 insertions, 5 deletions
diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c
index 8831b2a0303a..45becfb1ffe9 100644
--- a/fs/bcachefs/alloc_background.c
+++ b/fs/bcachefs/alloc_background.c
@@ -354,6 +354,7 @@ static int bch2_alloc_read_fn(struct btree_trans *trans, struct bkey_s_c k)
g = bucket(ca, k.k->p.offset);
u = bch2_alloc_unpack(k);
+ *bucket_gen(ca, k.k->p.offset) = u.gen;
g->_mark.gen = u.gen;
g->_mark.data_type = u.data_type;
g->_mark.dirty_sectors = u.dirty_sectors;
@@ -748,6 +749,7 @@ static int bch2_invalidate_one_bucket(struct bch_fs *c, struct bch_dev *ca,
!bucket_needs_journal_commit(m, c->journal.last_seq_ondisk)) {
BUG_ON(m.data_type);
bucket_cmpxchg(g, m, m.gen++);
+ *bucket_gen(ca, b) = m.gen;
percpu_up_read(&c->mark_lock);
goto out;
}
diff --git a/fs/bcachefs/alloc_foreground.c b/fs/bcachefs/alloc_foreground.c
index 7506d54c854b..e2038032b872 100644
--- a/fs/bcachefs/alloc_foreground.c
+++ b/fs/bcachefs/alloc_foreground.c
@@ -161,7 +161,7 @@ static void verify_not_stale(struct bch_fs *c, const struct open_buckets *obs)
open_bucket_for_each(c, obs, ob, i) {
struct bch_dev *ca = bch_dev_bkey_exists(c, ob->dev);
- BUG_ON(bucket(ca, ob->bucket)->mark.gen != ob->gen);
+ BUG_ON(*bucket_gen(ca, ob->bucket) != ob->gen);
}
rcu_read_unlock();
#endif
@@ -273,7 +273,7 @@ out:
ob->sectors_free = ca->mi.bucket_size;
ob->alloc_reserve = reserve;
ob->dev = ca->dev_idx;
- ob->gen = bucket(ca, b)->mark.gen;
+ ob->gen = *bucket_gen(ca, b);
ob->bucket = b;
spin_unlock(&ob->lock);
diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h
index 6c686be28b39..c282086079fb 100644
--- a/fs/bcachefs/bcachefs.h
+++ b/fs/bcachefs/bcachefs.h
@@ -445,6 +445,7 @@ struct bch_dev {
* Or rcu_read_lock(), but only for ptr_stale():
*/
struct bucket_array __rcu *buckets[2];
+ struct bucket_gens *bucket_gens;
unsigned long *buckets_nouse;
struct rw_semaphore bucket_lock;
diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c
index c1b0d0be07a6..4b7fe4a5def9 100644
--- a/fs/bcachefs/buckets.c
+++ b/fs/bcachefs/buckets.c
@@ -535,6 +535,20 @@ void bch2_mark_alloc_bucket(struct bch_fs *c, struct bch_dev *ca,
BUG_ON(owned_by_allocator == old.owned_by_allocator);
}
+static inline u8 bkey_alloc_gen(struct bkey_s_c k)
+{
+ switch (k.k->type) {
+ case KEY_TYPE_alloc:
+ return bkey_s_c_to_alloc(k).v->gen;
+ case KEY_TYPE_alloc_v2:
+ return bkey_s_c_to_alloc_v2(k).v->gen;
+ case KEY_TYPE_alloc_v3:
+ return bkey_s_c_to_alloc_v3(k).v->gen;
+ default:
+ return 0;
+ }
+}
+
static int bch2_mark_alloc(struct btree_trans *trans,
struct bkey_s_c old, struct bkey_s_c new,
unsigned flags)
@@ -573,9 +587,13 @@ static int bch2_mark_alloc(struct btree_trans *trans,
if (new.k->p.offset >= ca->mi.nbuckets)
return 0;
+ u = bch2_alloc_unpack(new);
+
percpu_down_read(&c->mark_lock);
+ if (!gc && u.gen != bkey_alloc_gen(old))
+ *bucket_gen(ca, new.k->p.offset) = u.gen;
+
g = __bucket(ca, new.k->p.offset, gc);
- u = bch2_alloc_unpack(new);
old_m = bucket_cmpxchg(g, m, ({
m.gen = u.gen;
@@ -2131,9 +2149,18 @@ static void buckets_free_rcu(struct rcu_head *rcu)
buckets->nbuckets * sizeof(struct bucket));
}
+static void bucket_gens_free_rcu(struct rcu_head *rcu)
+{
+ struct bucket_gens *buckets =
+ container_of(rcu, struct bucket_gens, rcu);
+
+ kvpfree(buckets, sizeof(struct bucket_array) + buckets->nbuckets);
+}
+
int bch2_dev_buckets_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets)
{
struct bucket_array *buckets = NULL, *old_buckets = NULL;
+ struct bucket_gens *bucket_gens = NULL, *old_bucket_gens = NULL;
unsigned long *buckets_nouse = NULL;
alloc_fifo free[RESERVE_NR];
alloc_fifo free_inc;
@@ -2157,6 +2184,8 @@ int bch2_dev_buckets_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets)
if (!(buckets = kvpmalloc(sizeof(struct bucket_array) +
nbuckets * sizeof(struct bucket),
GFP_KERNEL|__GFP_ZERO)) ||
+ !(bucket_gens = kvpmalloc(sizeof(struct bucket_gens) + nbuckets,
+ GFP_KERNEL|__GFP_ZERO)) ||
!(buckets_nouse = kvpmalloc(BITS_TO_LONGS(nbuckets) *
sizeof(unsigned long),
GFP_KERNEL|__GFP_ZERO)) ||
@@ -2169,6 +2198,8 @@ int bch2_dev_buckets_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets)
buckets->first_bucket = ca->mi.first_bucket;
buckets->nbuckets = nbuckets;
+ bucket_gens->first_bucket = ca->mi.first_bucket;
+ bucket_gens->nbuckets = nbuckets;
bch2_copygc_stop(c);
@@ -2179,6 +2210,7 @@ int bch2_dev_buckets_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets)
}
old_buckets = bucket_array(ca);
+ old_bucket_gens = rcu_dereference_protected(ca->bucket_gens, 1);
if (resize) {
size_t n = min(buckets->nbuckets, old_buckets->nbuckets);
@@ -2186,13 +2218,18 @@ int bch2_dev_buckets_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets)
memcpy(buckets->b,
old_buckets->b,
n * sizeof(struct bucket));
+ memcpy(bucket_gens->b,
+ old_bucket_gens->b,
+ n);
memcpy(buckets_nouse,
ca->buckets_nouse,
BITS_TO_LONGS(n) * sizeof(unsigned long));
}
rcu_assign_pointer(ca->buckets[0], buckets);
- buckets = old_buckets;
+ rcu_assign_pointer(ca->bucket_gens, bucket_gens);
+ buckets = old_buckets;
+ bucket_gens = old_bucket_gens;
swap(ca->buckets_nouse, buckets_nouse);
@@ -2226,6 +2263,8 @@ err:
free_fifo(&free[i]);
kvpfree(buckets_nouse,
BITS_TO_LONGS(nbuckets) * sizeof(unsigned long));
+ if (bucket_gens)
+ call_rcu(&old_buckets->rcu, bucket_gens_free_rcu);
if (buckets)
call_rcu(&old_buckets->rcu, buckets_free_rcu);
diff --git a/fs/bcachefs/buckets.h b/fs/bcachefs/buckets.h
index 61baaa66b6b5..6eeb95068b3b 100644
--- a/fs/bcachefs/buckets.h
+++ b/fs/bcachefs/buckets.h
@@ -63,6 +63,24 @@ static inline struct bucket *bucket(struct bch_dev *ca, size_t b)
return __bucket(ca, b, false);
}
+static inline struct bucket_gens *bucket_gens(struct bch_dev *ca)
+{
+ return rcu_dereference_check(ca->bucket_gens,
+ !ca->fs ||
+ percpu_rwsem_is_held(&ca->fs->mark_lock) ||
+ lockdep_is_held(&ca->fs->gc_lock) ||
+ lockdep_is_held(&ca->bucket_lock));
+
+}
+
+static inline u8 *bucket_gen(struct bch_dev *ca, size_t b)
+{
+ struct bucket_gens *gens = bucket_gens(ca);
+
+ BUG_ON(b < gens->first_bucket || b >= gens->nbuckets);
+ return gens->b + b;
+}
+
/*
* bucket_gc_gen() returns the difference between the bucket's current gen and
* the oldest gen of any pointer into that bucket in the btree.
@@ -123,7 +141,7 @@ static inline u8 ptr_stale(struct bch_dev *ca,
u8 ret;
rcu_read_lock();
- ret = gen_after(PTR_BUCKET(ca, ptr)->mark.gen, ptr->gen);
+ ret = gen_after(*bucket_gen(ca, PTR_BUCKET_NR(ca, ptr)), ptr->gen);
rcu_read_unlock();
return ret;
diff --git a/fs/bcachefs/buckets_types.h b/fs/bcachefs/buckets_types.h
index b2de2995c5e7..18bca269b750 100644
--- a/fs/bcachefs/buckets_types.h
+++ b/fs/bcachefs/buckets_types.h
@@ -52,6 +52,13 @@ struct bucket_array {
struct bucket b[];
};
+struct bucket_gens {
+ struct rcu_head rcu;
+ u16 first_bucket;
+ size_t nbuckets;
+ u8 b[];
+};
+
struct bch_dev_usage {
u64 buckets_ec;
u64 buckets_unavailable;
generated by cgit v1.2.3 (git 2.39.0) at 2025-03-15 12:47:14 +0300