bcachefs: "buckets with backpointer mismatches" now allocated on demand

More self healing work: we're going to be calling
check_bucket_backpointer_mismatch() at runtime, outside of fsck.

Then when we need to we'll kick off the full
check_extents_to_backpointers recovery pass.

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

fs/bcachefs/backpointers.c

@@ -15,6 +15,14 @@
 
 #include <linux/mm.h>
 
+static inline struct bbpos bp_to_bbpos(struct bch_backpointer bp)
+{
+	return (struct bbpos) {
+		.btree	= bp.btree_id,
+		.pos	= bp.pos,
+	};
+}
+
 int bch2_backpointer_validate(struct bch_fs *c, struct bkey_s_c k,
 			      struct bkey_validate_context from)
 {
@@ -671,8 +679,22 @@ static int check_extent_to_backpointers(struct btree_trans *trans,
 
 		rcu_read_lock();
 		struct bch_dev *ca = bch2_dev_rcu_noerror(c, p.ptr.dev);
-		bool check = ca && test_bit(PTR_BUCKET_NR(ca, &p.ptr), ca->bucket_backpointer_mismatches);
-		bool empty = ca && test_bit(PTR_BUCKET_NR(ca, &p.ptr), ca->bucket_backpointer_empty);
+		if (!ca) {
+			rcu_read_unlock();
+			continue;
+		}
+
+		u64 b = PTR_BUCKET_NR(ca, &p.ptr);
+		bool set[2];
+
+		for (unsigned i = 0; i < ARRAY_SIZE(ca->bucket_backpointer_mismatches); i++) {
+			unsigned long *bitmap =
+				READ_ONCE(ca->bucket_backpointer_mismatches[i].buckets);
+			set[i] = bitmap && test_bit(b, bitmap);
+		}
+
+		bool check = set[0];
+		bool empty = set[1];
 
 		bool stale = p.ptr.cached && (!ca || dev_ptr_stale_rcu(ca, &p.ptr));
 		rcu_read_unlock();
@@ -724,14 +746,6 @@ static int check_btree_root_to_backpointers(struct btree_trans *trans,
 	return ret;
 }
 
-static inline struct bbpos bp_to_bbpos(struct bch_backpointer bp)
-{
-	return (struct bbpos) {
-		.btree	= bp.btree_id,
-		.pos	= bp.pos,
-	};
-}
-
 static u64 mem_may_pin_bytes(struct bch_fs *c)
 {
 	struct sysinfo i;
@@ -933,12 +947,25 @@ static int check_bucket_backpointer_mismatch(struct btree_trans *trans, struct b
 			goto err;
 		}
 
-		if (!sectors[ALLOC_dirty] &&
-		    !sectors[ALLOC_stripe] &&
-		    !sectors[ALLOC_cached])
-			__set_bit(alloc_k.k->p.offset, ca->bucket_backpointer_empty);
-		else
-			__set_bit(alloc_k.k->p.offset, ca->bucket_backpointer_mismatches);
+		bool empty = (sectors[ALLOC_dirty] +
+			      sectors[ALLOC_stripe] +
+			      sectors[ALLOC_cached]) == 0;
+
+		struct bucket_bitmap *bitmap = &ca->bucket_backpointer_mismatches[empty];
+
+		mutex_lock(&bitmap->lock);
+		if (!bitmap->buckets) {
+			bitmap->buckets = kvcalloc(BITS_TO_LONGS(ca->mi.nbuckets),
+						   sizeof(unsigned long), GFP_KERNEL);
+			if (!bitmap->buckets) {
+				mutex_unlock(&bitmap->lock);
+				ret = -BCH_ERR_ENOMEM_backpointer_mismatches_bitmap;
+				goto err;
+			}
+		}
+
+		bitmap->nr += !__test_and_set_bit(alloc_k.k->p.offset, bitmap->buckets);
+		mutex_unlock(&bitmap->lock);
 	}
 err:
 	bch2_dev_put(ca);
@@ -962,8 +989,19 @@ static bool backpointer_node_has_missing(struct bch_fs *c, struct bkey_s_c k)
 				goto next;
 
 			struct bpos bucket = bp_pos_to_bucket(ca, pos);
-			bucket.offset = find_next_bit(ca->bucket_backpointer_mismatches,
-						      ca->mi.nbuckets, bucket.offset);
+			u64 next = ca->mi.nbuckets;
+
+			for (unsigned i = 0; i < ARRAY_SIZE(ca->bucket_backpointer_mismatches); i++) {
+				unsigned long *bitmap =
+					READ_ONCE(ca->bucket_backpointer_mismatches[i].buckets);
+				if (bitmap)
+					next = min_t(u64, next,
+						     find_next_bit(bitmap,
+								   ca->mi.nbuckets,
+								   bucket.offset));
+			}
+
+			bucket.offset = next;
 			if (bucket.offset == ca->mi.nbuckets)
 				goto next;
 
@@ -1072,28 +1110,6 @@ int bch2_check_extents_to_backpointers(struct bch_fs *c)
 {
 	int ret = 0;
 
-	/*
-	 * Can't allow devices to come/go/resize while we have bucket bitmaps
-	 * allocated
-	 */
-	down_read(&c->state_lock);
-
-	for_each_member_device(c, ca) {
-		BUG_ON(ca->bucket_backpointer_mismatches);
-		ca->bucket_backpointer_mismatches = kvcalloc(BITS_TO_LONGS(ca->mi.nbuckets),
-							     sizeof(unsigned long),
-							     GFP_KERNEL);
-		ca->bucket_backpointer_empty = kvcalloc(BITS_TO_LONGS(ca->mi.nbuckets),
-							sizeof(unsigned long),
-							GFP_KERNEL);
-		if (!ca->bucket_backpointer_mismatches ||
-		    !ca->bucket_backpointer_empty) {
-			bch2_dev_put(ca);
-			ret = -BCH_ERR_ENOMEM_backpointer_mismatches_bitmap;
-			goto err_free_bitmaps;
-		}
-	}
-
 	struct btree_trans *trans = bch2_trans_get(c);
 	struct extents_to_bp_state s = { .bp_start = POS_MIN };
 
@@ -1110,8 +1126,8 @@ int bch2_check_extents_to_backpointers(struct bch_fs *c)
 	u64 nr_buckets = 0, nr_mismatches = 0, nr_empty = 0;
 	for_each_member_device(c, ca) {
 		nr_buckets	+= ca->mi.nbuckets;
-		nr_mismatches	+= bitmap_weight(ca->bucket_backpointer_mismatches, ca->mi.nbuckets);
-		nr_empty	+= bitmap_weight(ca->bucket_backpointer_empty, ca->mi.nbuckets);
+		nr_mismatches	+= ca->bucket_backpointer_mismatches[0].nr;
+		nr_empty	+= ca->bucket_backpointer_mismatches[1].nr;
 	}
 
 	if (!nr_mismatches && !nr_empty)
@@ -1153,19 +1169,17 @@ int bch2_check_extents_to_backpointers(struct bch_fs *c)
 	bch2_trans_put(trans);
 	bch2_bkey_buf_exit(&s.last_flushed, c);
 	bch2_btree_cache_unpin(c);
-err_free_bitmaps:
-	for_each_member_device(c, ca) {
-		kvfree(ca->bucket_backpointer_empty);
-		ca->bucket_backpointer_empty = NULL;
-		kvfree(ca->bucket_backpointer_mismatches);
-		ca->bucket_backpointer_mismatches = NULL;
-	}
 
-	up_read(&c->state_lock);
+	for_each_member_device(c, ca)
+		for (unsigned i = 0; i < ARRAY_SIZE(ca->bucket_backpointer_mismatches); i++)
+			bch2_bucket_bitmap_free(&ca->bucket_backpointer_mismatches[i]);
+
 	bch_err_fn(c, ret);
 	return ret;
 }
 
+/* backpointers -> extents */
+
 static int check_one_backpointer(struct btree_trans *trans,
 				 struct bbpos start,
 				 struct bbpos end,
@@ -1281,3 +1295,12 @@ int bch2_check_backpointers_to_extents(struct bch_fs *c)
 	bch_err_fn(c, ret);
 	return ret;
 }
+
+void bch2_bucket_bitmap_free(struct bucket_bitmap *b)
+{
+	mutex_lock(&b->lock);
+	kvfree(b->buckets);
+	b->buckets = NULL;
+	b->nr	= 0;
+	mutex_unlock(&b->lock);
+}

fs/bcachefs/backpointers.h

@@ -182,8 +182,12 @@ struct bkey_s_c bch2_backpointer_get_key(struct btree_trans *, struct bkey_s_c_b
 struct btree *bch2_backpointer_get_node(struct btree_trans *, struct bkey_s_c_backpointer,
 					struct btree_iter *, struct bkey_buf *);
 
+int bch2_check_bucket_backpointer_mismatch(struct btree_trans *, struct bpos, struct bkey_buf *);
+
 int bch2_check_btree_backpointers(struct bch_fs *);
 int bch2_check_extents_to_backpointers(struct bch_fs *);
 int bch2_check_backpointers_to_extents(struct bch_fs *);
 
+void bch2_bucket_bitmap_free(struct bucket_bitmap *);
+
 #endif /* _BCACHEFS_BACKPOINTERS_BACKGROUND_H */

fs/bcachefs/bcachefs.h

@@ -574,6 +574,12 @@ enum bch_dev_write_ref {
 	BCH_DEV_WRITE_REF_NR,
 };
 
+struct bucket_bitmap {
+	unsigned long		*buckets;
+	u64			nr;
+	struct mutex		lock;
+};
+
 struct bch_dev {
 	struct kobject		kobj;
 #ifdef CONFIG_BCACHEFS_DEBUG
@@ -618,8 +624,7 @@ struct bch_dev {
 	u8			*oldest_gen;
 	unsigned long		*buckets_nouse;
 
-	unsigned long		*bucket_backpointer_mismatches;
-	unsigned long		*bucket_backpointer_empty;
+	struct bucket_bitmap	bucket_backpointer_mismatches[2];
 
 	struct bch_dev_usage_full __percpu
 				*usage;

fs/bcachefs/buckets.c

@@ -1324,6 +1324,28 @@ int bch2_dev_buckets_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets)
 		       sizeof(bucket_gens->b[0]) * copy);
 	}
 
+	for (unsigned i = 0; i < ARRAY_SIZE(ca->bucket_backpointer_mismatches); i++) {
+		struct bucket_bitmap *bitmap = &ca->bucket_backpointer_mismatches[i];
+
+		mutex_lock(&bitmap->lock);
+		if (bitmap->buckets) {
+			unsigned long *n = kvcalloc(BITS_TO_LONGS(nbuckets),
+						    sizeof(unsigned long), GFP_KERNEL);
+			if (!n) {
+				mutex_unlock(&bitmap->lock);
+				ret = -BCH_ERR_ENOMEM_backpointer_mismatches_bitmap;
+				goto err;
+			}
+
+			memcpy(n, bitmap->buckets,
+			       BITS_TO_LONGS(ca->mi.nbuckets) * sizeof(unsigned long));
+			kvfree(bitmap->buckets);
+			bitmap->buckets = n;
+
+		}
+		mutex_unlock(&bitmap->lock);
+	}
+
 	rcu_assign_pointer(ca->bucket_gens, bucket_gens);
 	bucket_gens	= old_bucket_gens;
 

fs/bcachefs/super.c

@@ -11,6 +11,7 @@
 #include "alloc_background.h"
 #include "alloc_foreground.h"
 #include "async_objs.h"
+#include "backpointers.h"
 #include "bkey_sort.h"
 #include "btree_cache.h"
 #include "btree_gc.h"
@@ -1341,6 +1342,9 @@ static void bch2_dev_free(struct bch_dev *ca)
 	if (ca->kobj.state_in_sysfs)
 		kobject_del(&ca->kobj);
 
+	for (unsigned i = 0; i < ARRAY_SIZE(ca->bucket_backpointer_mismatches); i++)
+		bch2_bucket_bitmap_free(&ca->bucket_backpointer_mismatches[i]);
+
 	bch2_free_super(&ca->disk_sb);
 	bch2_dev_allocator_background_exit(ca);
 	bch2_dev_journal_exit(ca);
@@ -1471,6 +1475,9 @@ static struct bch_dev *__bch2_dev_alloc(struct bch_fs *c,
 	atomic_long_set(&ca->ref, 1);
 #endif
 
+	for (unsigned i = 0; i < ARRAY_SIZE(ca->bucket_backpointer_mismatches); i++)
+		mutex_init(&ca->bucket_backpointer_mismatches[i].lock);
+
 	bch2_dev_allocator_background_init(ca);
 
 	if (enumerated_ref_init(&ca->io_ref[READ],  BCH_DEV_READ_REF_NR,  NULL) ||