github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-22 12:44:03 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge tag 'md-7.3-20260809' of https://git.kernel.org/pub/scm/linux/kernel/git/mdraid/linux into for-7.3/block

Browse Source 
Pull MD updates from Yu Kuai:

"This pull request contains:

 Bug Fixes:
 - Protect RAID5 bitmap batching, stripe-cache limits, and reshape
   recovery state; avoid failed-device reshape deadlocks, discard hangs,
   and PPL use-after-free. (Chen Cheng, Genjian Zhang, Sajal Gupta)
 - Recheck spare changes under array suspension before sync to avoid
   racing device removal. (Abd-Alrhman Masalkhi)
 - Fix RAID1 atomic-write constraints, serialized-device setup, and
   takeover I/O freezes. (Abd-Alrhman Masalkhi, Martin Wilck,
   Bruce Johnston)
 - Fix RAID10 atomic-write failure handling and reshape pool/bio
   lifetime bugs. (Abd-Alrhman Masalkhi, Chen Cheng)
- Fix bitmap error recovery, flush/sync accounting, reclaim safety,
   teardown, timer, use-after-free, and empty-range bugs, plus stale
   clone I/O accounting. (Chen Cheng, Yu Kuai)
 - Reject zero-sector RAID5 reshape chunks and correctly round bitmap
   ranges for non-power-of-two stripe widths. (Yu Kuai)
 - Prevent PF_MEMALLOC_NOIO state from leaking across tasks.
   (Chen Cheng)
 - Validate bad-block-log shift bounds and skip discard on unsupported
   member devices. (Coly Li, Wale Zhang)
 - Prevent RAID10 recovery corruption and large-array resync soft
   lockups. (Yunye Zhao)

 Improvements:
 - Add lockless bitmap reshape support for RAID5 and RAID10, including
   exact old/new mapping, cache growth, geometry lifecycle, checkpoint
   remapping, and bio splitting. (Yu Kuai)

 Cleanups:
 - Make RAID1 sequential-read hint accesses explicit to suppress false
   KCSAN reports. (Chen Cheng)
 - Remove redundant RAID10 barrier handling and align badblock range
   types. (Abd-Alrhman Masalkhi, Hiroshi Nishida)"

* tag 'md-7.3-20260809' of https://git.kernel.org/pub/scm/linux/kernel/git/mdraid/linux: (53 commits)
  md/raid1: don't set array_frozen in raid1_takeover()
  md: skip discard on unsupported member devices
  md: add cond_resched() to md_do_sync()'s skip path
  md/raid10: fix still_degraded being inverted in raid10_sync_request()
  md/raid5: split reshape bios before bitmap accounting
  md/raid5: wire llbitmap reshape lifecycle
  md/raid5: reject llbitmap reshape when md chunk shrinks
  md/raid5: add exact old and new llbitmap mapping helpers
  md/raid10: split reshape bios before bitmap accounting
  md/raid10: wire llbitmap reshape lifecycle
  md/raid10: reject llbitmap reshape when md chunk shrinks
  md/md-llbitmap: clamp state-machine walks to tracked bits
  md/md-llbitmap: remap checkpointed bits as reshape progresses
  md/md-llbitmap: don't skip reshape ranges from bitmap state
  md/md-llbitmap: add reshape range mapping helpers
  md/md-llbitmap: refuse reshape while llbitmap still needs sync
  md/md-llbitmap: finish reshape geometry
  md/md-llbitmap: track target reshape geometry fields
  md/md-llbitmap: grow the page cache in place for reshape
  md/md-llbitmap: allocate page controls independently
  ...
This commit is contained in:
Jens Axboe 2026-08-15 17:37:12 -06:00
commit 069f4aad34
13 changed files with 1279 additions and 402 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
drivers/md/md-bitmap.c
View File

@ -1730,6 +1730,13 @@ static void bitmap_start_write(struct mddev *mddev, sector_t offset,
}
}
static void bitmap_prepare_range(struct mddev *mddev, sector_t *offset,
unsigned long *sectors, bool discard)
{
if (mddev->pers->bitmap_sector)
mddev->pers->bitmap_sector(mddev, offset, sectors);
}
static void bitmap_end_write(struct mddev *mddev, sector_t offset,
unsigned long sectors)
{
@ -2064,23 +2071,18 @@ static void bitmap_end_behind_write(struct mddev *mddev)
bitmap->mddev->bitmap_info.max_write_behind);
}
static bool bitmap_wait_behind_writes(struct mddev *mddev, bool nowait)
static void bitmap_wait_behind_writes(struct mddev *mddev)
{
struct bitmap *bitmap = mddev->bitmap;
/* wait for behind writes to complete */
if (bitmap && atomic_read(&bitmap->behind_writes) > 0) {
if (nowait)
return false;
pr_debug("md:%s: behind writes in progress - waiting to stop.\n",
mdname(mddev));
/* need to kick something here to make sure I/O goes? */
wait_event(bitmap->behind_wait,
atomic_read(&bitmap->behind_writes) == 0);
}
return true;
}
static void bitmap_destroy(struct mddev *mddev)
@ -2090,7 +2092,7 @@ static void bitmap_destroy(struct mddev *mddev)
if (!bitmap) /* there was no bitmap */
return;
bitmap_wait_behind_writes(mddev, false);
bitmap_wait_behind_writes(mddev);
if (!test_bit(MD_SERIALIZE_POLICY, &mddev->flags))
mddev_destroy_serial_pool(mddev, NULL);
@ -2629,10 +2631,12 @@ static ssize_t
location_store(struct mddev *mddev, const char *buf, size_t len)
{
int rv;
unsigned int noio_flags;
rv = mddev_suspend_and_lock(mddev);
if (rv)
return rv;
noio_flags = memalloc_noio_save();
if (mddev->pers) {
if (mddev->recovery || mddev->sync_thread) {
@ -2719,6 +2723,7 @@ location_store(struct mddev *mddev, const char *buf, size_t len)
}
rv = 0;
out:
memalloc_noio_restore(noio_flags);
mddev_unlock_and_resume(mddev);
if (rv)
return rv;
@ -2862,7 +2867,7 @@ backlog_store(struct mddev *mddev, const char *buf, size_t len)
if (!has_write_mostly) {
pr_warn_ratelimited("%s: can't set backlog, no write mostly device available\n",
mdname(mddev));
mddev_unlock(mddev);
mddev_unlock_and_resume(mddev);
return -EINVAL;
}
@ -3083,6 +3088,7 @@ static struct bitmap_operations bitmap_ops = {
.flush = bitmap_flush,
.write_all = bitmap_write_all,
.dirty_bits = bitmap_dirty_bits,
.prepare_range = bitmap_prepare_range,
.unplug = bitmap_unplug,
.daemon_work = bitmap_daemon_work,

12
drivers/md/md-bitmap.h
View File

@ -29,6 +29,7 @@ enum bitmap_state {
BITMAP_FIRST_USE = 3, /* llbitmap is just created */
BITMAP_CLEAN = 4, /* llbitmap is created with assume_clean */
BITMAP_DAEMON_BUSY = 5, /* llbitmap daemon is not finished after daemon_sleep */
BITMAP_SHUTDOWN = 6, /* llbitmap is being destroyed */
BITMAP_HOSTENDIAN =15,
};
@ -93,12 +94,21 @@ struct bitmap_operations {
void (*write_all)(struct mddev *mddev);
void (*dirty_bits)(struct mddev *mddev, unsigned long s,
unsigned long e);
/* Prepare a range for this bitmap implementation. */
void (*prepare_range)(struct mddev *mddev,
sector_t *offset,
unsigned long *sectors,
bool discard);
void (*reshape_finish)(struct mddev *mddev);
int (*reshape_can_start)(struct mddev *mddev);
void (*reshape_mark)(struct mddev *mddev, sector_t old_pos,
sector_t new_pos);
void (*unplug)(struct mddev *mddev, bool sync);
void (*daemon_work)(struct mddev *mddev);
void (*start_behind_write)(struct mddev *mddev);
void (*end_behind_write)(struct mddev *mddev);
bool (*wait_behind_writes)(struct mddev *mddev, bool nowait);
void (*wait_behind_writes)(struct mddev *mddev);
md_bitmap_fn *start_write;
md_bitmap_fn *end_write;

1
drivers/md/md-linear.c
View File

@ -71,6 +71,7 @@ static int linear_set_limits(struct mddev *mddev)
int err;
md_init_stacking_limits(&lim);
lim.features |= BLK_FEAT_NOWAIT;
lim.max_hw_sectors = mddev->chunk_sectors;
lim.logical_block_size = mddev->logical_block_size;
lim.max_write_zeroes_sectors = mddev->chunk_sectors;

754
drivers/md/md-llbitmap.c
View File

File diff suppressed because it is too large Load Diff

178
drivers/md/md.c
View File

@ -233,23 +233,22 @@ static int rdev_need_serial(struct md_rdev *rdev)
void mddev_create_serial_pool(struct mddev *mddev, struct md_rdev *rdev)
{
int ret = 0;
unsigned int noio_flags;
if (rdev && !rdev_need_serial(rdev) &&
if (!test_bit(MD_SERIALIZE_POLICY, &mddev->flags) &&
rdev && !rdev_need_serial(rdev) &&
!test_bit(CollisionCheck, &rdev->flags))
return;
noio_flags = memalloc_noio_save();
if (!rdev)
ret = rdevs_init_serial(mddev);
else
ret = rdev_init_serial(rdev);
if (ret)
return;
goto out;
if (mddev->serial_info_pool == NULL) {
/*
* already in memalloc noio context by
* mddev_suspend()
*/
mddev->serial_info_pool =
mempool_create_kmalloc_pool(NR_SERIAL_INFOS,
sizeof(struct serial_info));
@ -258,6 +257,8 @@ void mddev_create_serial_pool(struct mddev *mddev, struct md_rdev *rdev)
pr_err("can't alloc memory pool for serialization\n");
}
}
out:
memalloc_noio_restore(noio_flags);
}
/*
@ -516,9 +517,6 @@ int mddev_suspend(struct mddev *mddev, bool interruptible)
*/
WRITE_ONCE(mddev->suspended, mddev->suspended + 1);
/* restrict memory reclaim I/O during raid array is suspend */
mddev->noio_flag = memalloc_noio_save();
mutex_unlock(&mddev->suspend_mutex);
return 0;
}
@ -535,9 +533,6 @@ static void __mddev_resume(struct mddev *mddev, bool recovery_needed)
return;
}
/* entred the memalloc scope from mddev_suspend() */
memalloc_noio_restore(mddev->noio_flag);
percpu_ref_resurrect(&mddev->active_io);
wake_up(&mddev->sb_wait);
@ -1920,6 +1915,13 @@ static int super_1_load(struct md_rdev *rdev, struct md_rdev *refdev, int minor_
rdev->bb_page, REQ_OP_READ, true))
return -EIO;
bbp = (__le64 *)page_address(rdev->bb_page);
/* check for badblocks api. */
if (sb->bblog_shift >= BITS_PER_TYPE(sector_t)) {
pr_err("md: %pg: bogus bblog_shift %u for badblocks.\n",
rdev->bdev, sb->bblog_shift);
return -EINVAL;
}
rdev->badblocks.shift = sb->bblog_shift;
for (i = 0 ; i < (sectors << (9-3)) ; i++, bbp++) {
u64 bb = le64_to_cpu(*bbp);
@ -4047,6 +4049,7 @@ level_store(struct mddev *mddev, const char *buf, size_t len)
char clevel[16];
ssize_t rv;
size_t slen = len;
unsigned int noio_flags;
struct md_personality *pers, *oldpers;
long level;
void *priv, *oldpriv;
@ -4058,6 +4061,7 @@ level_store(struct mddev *mddev, const char *buf, size_t len)
rv = mddev_suspend_and_lock(mddev);
if (rv)
return rv;
noio_flags = memalloc_noio_save();
if (mddev->pers == NULL) {
memcpy(mddev->clevel, buf, slen);
@ -4233,6 +4237,7 @@ level_store(struct mddev *mddev, const char *buf, size_t len)
md_new_event();
rv = len;
out_unlock:
memalloc_noio_restore(noio_flags);
mddev_unlock_and_resume(mddev);
return rv;
}
@ -4412,6 +4417,7 @@ static ssize_t
raid_disks_store(struct mddev *mddev, const char *buf, size_t len)
{
unsigned int n;
unsigned int noio_flags;
int err;
err = kstrtouint(buf, 10, &n);
@ -4421,6 +4427,7 @@ raid_disks_store(struct mddev *mddev, const char *buf, size_t len)
err = mddev_suspend_and_lock(mddev);
if (err)
return err;
noio_flags = memalloc_noio_save();
if (mddev->pers) {
if (n != mddev->raid_disks)
err = update_raid_disks(mddev, n);
@ -4444,6 +4451,7 @@ raid_disks_store(struct mddev *mddev, const char *buf, size_t len)
} else
mddev->raid_disks = n;
out_unlock:
memalloc_noio_restore(noio_flags);
mddev_unlock_and_resume(mddev);
return err ? err : len;
}
@ -4824,6 +4832,7 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len)
int minor;
dev_t dev;
struct md_rdev *rdev;
unsigned int noio_flags;
int err;
if (!*buf || *e != ':' || !e[1] || e[1] == '\n')
@ -4839,6 +4848,7 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len)
err = mddev_suspend_and_lock(mddev);
if (err)
return err;
noio_flags = memalloc_noio_save();
if (mddev->persistent) {
rdev = md_import_device(dev, mddev->major_version,
mddev->minor_version);
@ -4857,6 +4867,7 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len)
rdev = md_import_device(dev, -1, -1);
if (IS_ERR(rdev)) {
memalloc_noio_restore(noio_flags);
mddev_unlock_and_resume(mddev);
return PTR_ERR(rdev);
}
@ -4864,6 +4875,7 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len)
out:
if (err)
export_rdev(rdev);
memalloc_noio_restore(noio_flags);
mddev_unlock_and_resume(mddev);
if (!err)
md_new_event();
@ -5263,21 +5275,28 @@ action_store(struct mddev *mddev, const char *page, size_t len)
if (!mddev->pers || !mddev->pers->sync_request)
return -EINVAL;
action = md_sync_action_by_name(page);
if (action == ACTION_RESHAPE) {
ret = mddev_suspend(mddev, true);
if (ret)
return ret;
}
retry:
if (work_busy(&mddev->sync_work))
flush_work(&mddev->sync_work);
ret = mddev_lock(mddev);
if (ret)
if (ret) {
if (action == ACTION_RESHAPE)
mddev_resume(mddev);
return ret;
}
if (work_busy(&mddev->sync_work)) {
mddev_unlock(mddev);
goto retry;
}
action = md_sync_action_by_name(page);
/* TODO: mdadm rely on "idle" to start sync_thread. */
if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) {
switch (action) {
@ -5347,6 +5366,8 @@ action_store(struct mddev *mddev, const char *page, size_t len)
out:
mddev_unlock(mddev);
if (action == ACTION_RESHAPE)
mddev_resume(mddev);
return ret;
}
@ -6283,7 +6304,7 @@ void md_init_stacking_limits(struct queue_limits *lim)
{
blk_set_stacking_limits(lim);
lim->features = BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA |
BLK_FEAT_IO_STAT | BLK_FEAT_NOWAIT;
BLK_FEAT_IO_STAT;
}
EXPORT_SYMBOL_GPL(md_init_stacking_limits);
@ -6631,7 +6652,6 @@ int md_run(struct mddev *mddev)
int err;
struct md_rdev *rdev;
struct md_personality *pers;
bool nowait = true;
if (list_empty(&mddev->disks))
/* cannot run an array with no devices.. */
@ -6702,7 +6722,6 @@ int md_run(struct mddev *mddev)
}
}
sysfs_notify_dirent_safe(rdev->sysfs_state);
nowait = nowait && bdev_nowait(rdev->bdev);
}
pers = get_pers(mddev->level, mddev->clevel);
@ -7050,7 +7069,7 @@ EXPORT_SYMBOL_GPL(md_stop_writes);
static void mddev_detach(struct mddev *mddev)
{
if (md_bitmap_enabled(mddev, false))
mddev->bitmap_ops->wait_behind_writes(mddev, false);
mddev->bitmap_ops->wait_behind_writes(mddev);
if (mddev->pers && mddev->pers->quiesce && !is_md_suspended(mddev)) {
mddev->pers->quiesce(mddev, 1);
mddev->pers->quiesce(mddev, 0);
@ -7066,8 +7085,8 @@ static void __md_stop(struct mddev *mddev)
{
struct md_personality *pers = mddev->pers;
md_bitmap_destroy(mddev);
mddev_detach(mddev);
md_bitmap_destroy(mddev);
spin_lock(&mddev->lock);
mddev->pers = NULL;
spin_unlock(&mddev->lock);
@ -8324,8 +8343,10 @@ static int md_ioctl(struct block_device *bdev, blk_mode_t mode,
unsigned int cmd, unsigned long arg)
{
int err = 0;
unsigned int noio_flags = 0;
void __user *argp = (void __user *)arg;
struct mddev *mddev = NULL;
bool suspend;
err = md_ioctl_valid(cmd);
if (err)
@ -8375,13 +8396,15 @@ static int md_ioctl(struct block_device *bdev, blk_mode_t mode,
if (!md_is_rdwr(mddev))
flush_work(&mddev->sync_work);
err = md_ioctl_need_suspend(cmd) ? mddev_suspend_and_lock(mddev) :
mddev_lock(mddev);
suspend = md_ioctl_need_suspend(cmd);
err = suspend ? mddev_suspend_and_lock(mddev) : mddev_lock(mddev);
if (err) {
pr_debug("md: ioctl lock interrupted, reason %d, cmd %d\n",
err, cmd);
goto out;
}
if (suspend)
noio_flags = memalloc_noio_save();
if (cmd == SET_ARRAY_INFO) {
err = __md_set_array_info(mddev, argp);
@ -8506,8 +8529,12 @@ static int md_ioctl(struct block_device *bdev, blk_mode_t mode,
err != -EINVAL)
mddev->hold_active = 0;
md_ioctl_need_suspend(cmd) ? mddev_unlock_and_resume(mddev) :
mddev_unlock(mddev);
if (suspend) {
memalloc_noio_restore(noio_flags);
mddev_unlock_and_resume(mddev);
} else {
mddev_unlock(mddev);
}
out:
if (cmd == STOP_ARRAY_RO || (err && cmd == STOP_ARRAY))
@ -9350,6 +9377,10 @@ void md_submit_discard_bio(struct mddev *mddev, struct md_rdev *rdev,
{
struct bio *discard_bio = NULL;
/* Discard is optional, so silently skip members that do not support it. */
if (unlikely(!bdev_max_discard_sectors(rdev->bdev)))
return;
__blkdev_issue_discard(rdev->bdev, start, size, GFP_NOIO, &discard_bio);
if (!discard_bio)
return;
@ -9361,17 +9392,62 @@ void md_submit_discard_bio(struct mddev *mddev, struct md_rdev *rdev,
}
EXPORT_SYMBOL_GPL(md_submit_discard_bio);
struct bio *mddev_bio_split_at_reshape_offset(struct mddev *mddev,
struct bio *bio,
unsigned int *max_sectors,
struct bio_set *bs)
{
sector_t boundary;
sector_t start;
sector_t end;
unsigned int split_sectors;
split_sectors = bio_sectors(bio);
if (max_sectors && *max_sectors && *max_sectors < split_sectors)
split_sectors = *max_sectors;
if (!test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
goto split;
boundary = READ_ONCE(mddev->reshape_position);
start = bio->bi_iter.bi_sector;
end = bio_end_sector(bio);
if (start >= boundary || end <= boundary)
goto split;
if (boundary - start < split_sectors)
split_sectors = boundary - start;
split:
if (max_sectors)
*max_sectors = split_sectors;
if (split_sectors < bio_sectors(bio)) {
bio = bio_submit_split_bioset(bio, split_sectors, bs);
if (bio)
bio->bi_opf |= REQ_NOMERGE;
}
return bio;
}
EXPORT_SYMBOL_GPL(mddev_bio_split_at_reshape_offset);
static void md_bitmap_prepare_range(struct mddev *mddev, sector_t *offset,
unsigned long *sectors, bool discard)
{
mddev->bitmap_ops->prepare_range(mddev, offset, sectors, discard);
}
static void md_bitmap_start(struct mddev *mddev,
struct md_io_clone *md_io_clone)
{
md_bitmap_fn *fn = unlikely(md_io_clone->rw == STAT_DISCARD) ?
mddev->bitmap_ops->start_discard :
bool discard = md_io_clone->rw == STAT_DISCARD;
md_bitmap_fn *fn = discard ? mddev->bitmap_ops->start_discard :
mddev->bitmap_ops->start_write;
if (mddev->pers->bitmap_sector)
mddev->pers->bitmap_sector(mddev, &md_io_clone->offset,
&md_io_clone->sectors);
md_bitmap_prepare_range(mddev, &md_io_clone->offset,
&md_io_clone->sectors, discard);
if (!md_io_clone->sectors)
return;
fn(mddev, md_io_clone->offset, md_io_clone->sectors);
}
@ -9392,7 +9468,8 @@ static void md_end_clone_io(struct bio *bio)
struct mddev *mddev = md_io_clone->mddev;
struct completion *reshape_completion = bio->bi_private;
if (bio_data_dir(orig_bio) == WRITE && md_bitmap_enabled(mddev, false))
if (bio_data_dir(orig_bio) == WRITE && md_io_clone->sectors &&
md_bitmap_enabled(mddev, false))
md_bitmap_end(mddev, md_io_clone);
if (bio->bi_status && !orig_bio->bi_status)
@ -9419,10 +9496,14 @@ static void md_clone_bio(struct mddev *mddev, struct bio **bio)
md_io_clone = container_of(clone, struct md_io_clone, bio_clone);
md_io_clone->orig_bio = *bio;
md_io_clone->mddev = mddev;
md_io_clone->sectors = 0;
if (blk_queue_io_stat(bdev->bd_disk->queue))
md_io_clone->start_time = bio_start_io_acct(*bio);
else
md_io_clone->start_time = 0;
if (bio_data_dir(*bio) == WRITE && md_bitmap_enabled(mddev, false)) {
if (bio_data_dir(*bio) == WRITE && bio_sectors(*bio) &&
md_bitmap_enabled(mddev, false)) {
md_io_clone->offset = (*bio)->bi_iter.bi_sector;
md_io_clone->sectors = bio_sectors(*bio);
md_io_clone->rw = op_stat_group(bio_op(*bio));
@ -9881,8 +9962,10 @@ void md_do_sync(struct md_thread *thread)
*/
md_new_event();
if (last_check + window > io_sectors || j == max_sectors)
if (last_check + window > io_sectors || j == max_sectors) {
cond_resched();
continue;
}
last_check = io_sectors;
repeat:
@ -10175,19 +10258,34 @@ static void md_start_sync(struct work_struct *ws)
struct mddev *mddev = container_of(ws, struct mddev, sync_work);
int spares = 0;
bool suspend = false;
unsigned int noio_flags = 0;
char *name;
/*
* If reshape is still in progress, spares won't be added or removed
* from conf until reshape is done.
*/
if (mddev->reshape_position == MaxSector &&
if ((mddev->reshape_position == MaxSector || !md_is_rdwr(mddev)) &&
md_spares_need_change(mddev)) {
suspend = true;
mddev_suspend(mddev, false);
noio_flags = memalloc_noio_save();
}
mddev_lock_nointr(mddev);
/*
* The spare configuration can change before reconfig_mutex is acquired.
* Recheck while holding the lock and suspend if needed.
*/
if (!suspend && (mddev->reshape_position == MaxSector || !md_is_rdwr(mddev)) &&
md_spares_need_change(mddev)) {
mddev_unlock(mddev);
mddev_suspend_and_lock_nointr(mddev);
suspend = true;
noio_flags = memalloc_noio_save();
}
if (!md_is_rdwr(mddev)) {
/*
* On a read-only array we can:
@ -10231,8 +10329,10 @@ static void md_start_sync(struct work_struct *ws)
* https://bugzilla.kernel.org/show_bug.cgi?id=218200
* Therefore, use __mddev_resume(mddev, false).
*/
if (suspend)
if (suspend) {
memalloc_noio_restore(noio_flags);
__mddev_resume(mddev, false);
}
md_wakeup_thread(mddev->sync_thread);
sysfs_notify_dirent_safe(mddev->sysfs_action);
md_new_event();
@ -10251,8 +10351,10 @@ static void md_start_sync(struct work_struct *ws)
* https://bugzilla.kernel.org/show_bug.cgi?id=218200
* Therefore, use __mddev_resume(mddev, false).
*/
if (suspend)
if (suspend) {
memalloc_noio_restore(noio_flags);
__mddev_resume(mddev, false);
}
wake_up(&resync_wait);
if (test_and_clear_bit(MD_RECOVERY_RECOVER, &mddev->recovery) &&
@ -10553,7 +10655,7 @@ EXPORT_SYMBOL(md_finish_reshape);
/* Bad block management */
/* Returns true on success, false on failure */
bool rdev_set_badblocks(struct md_rdev *rdev, sector_t s, int sectors,
bool rdev_set_badblocks(struct md_rdev *rdev, sector_t s, sector_t sectors,
int is_new)
{
struct mddev *mddev = rdev->mddev;
@ -10593,7 +10695,7 @@ bool rdev_set_badblocks(struct md_rdev *rdev, sector_t s, int sectors,
}
EXPORT_SYMBOL_GPL(rdev_set_badblocks);
void rdev_clear_badblocks(struct md_rdev *rdev, sector_t s, int sectors,
void rdev_clear_badblocks(struct md_rdev *rdev, sector_t s, sector_t sectors,
int is_new)
{
if (is_new)

15
drivers/md/md.h
View File

@ -311,7 +311,7 @@ static inline int is_badblock(struct md_rdev *rdev, sector_t s, sector_t sectors
}
static inline int rdev_has_badblock(struct md_rdev *rdev, sector_t s,
int sectors)
sector_t sectors)
{
sector_t first_bad;
sector_t bad_sectors;
@ -319,9 +319,9 @@ static inline int rdev_has_badblock(struct md_rdev *rdev, sector_t s,
return is_badblock(rdev, s, sectors, &first_bad, &bad_sectors);
}
extern bool rdev_set_badblocks(struct md_rdev *rdev, sector_t s, int sectors,
extern bool rdev_set_badblocks(struct md_rdev *rdev, sector_t s, sector_t sectors,
int is_new);
extern void rdev_clear_badblocks(struct md_rdev *rdev, sector_t s, int sectors,
extern void rdev_clear_badblocks(struct md_rdev *rdev, sector_t s, sector_t sectors,
int is_new);
struct md_cluster_info;
struct md_cluster_operations;
@ -621,7 +621,6 @@ struct mddev {
struct md_cluster_info *cluster_info;
struct md_cluster_operations *cluster_ops;
unsigned int good_device_nr; /* good device num within cluster raid */
unsigned int noio_flag; /* for memalloc scope API */
/*
* Temporarily store rdev that will be finally removed when
@ -798,6 +797,10 @@ struct md_personality
/* convert io ranges from array to bitmap */
void (*bitmap_sector)(struct mddev *mddev, sector_t *offset,
unsigned long *sectors);
void (*bitmap_sector_map)(struct mddev *mddev, sector_t *offset,
unsigned long *sectors, bool previous);
sector_t (*bitmap_sync_size)(struct mddev *mddev, bool previous);
sector_t (*bitmap_array_sectors)(struct mddev *mddev, bool previous);
};
struct md_sysfs_entry {
@ -921,6 +924,10 @@ extern void md_error(struct mddev *mddev, struct md_rdev *rdev);
extern void md_finish_reshape(struct mddev *mddev);
void md_submit_discard_bio(struct mddev *mddev, struct md_rdev *rdev,
struct bio *bio, sector_t start, sector_t size);
struct bio *mddev_bio_split_at_reshape_offset(struct mddev *mddev,
struct bio *bio,
unsigned int *max_sectors,
struct bio_set *bs);
void md_account_bio(struct mddev *mddev, struct bio **bio);
extern bool __must_check md_flush_request(struct mddev *mddev, struct bio *bio);

1
drivers/md/raid0.c
View File

@ -385,6 +385,7 @@ static int raid0_set_limits(struct mddev *mddev)
int err;
md_init_stacking_limits(&lim);
lim.features |= BLK_FEAT_NOWAIT;
lim.max_hw_sectors = mddev->chunk_sectors;
lim.max_write_zeroes_sectors = mddev->chunk_sectors;
lim.max_hw_wzeroes_unmap_sectors = mddev->chunk_sectors;

8
drivers/md/raid1-10.c
View File

@ -290,9 +290,8 @@ static inline bool raid1_should_read_first(struct mddev *mddev,
}
/*
* bio with REQ_RAHEAD or REQ_NOWAIT can fail at anytime, before such IO is
* submitted to the underlying disks, hence don't record badblocks or retry
* in this case.
* bio with REQ_RAHEAD can fail at anytime, before such IO is submitted to the
* underlying disks, hence don't record badblocks or retry in this case.
*
* BLK_STS_INVAL means the bio was not valid for the underlying device. This
* is a user error, not a device failure, so retrying or recording bad blocks
@ -300,6 +299,5 @@ static inline bool raid1_should_read_first(struct mddev *mddev,
*/
static inline bool raid1_should_handle_error(struct bio *bio)
{
return !(bio->bi_opf & (REQ_RAHEAD | REQ_NOWAIT)) &&
bio->bi_status != BLK_STS_INVAL;
return !(bio->bi_opf & REQ_RAHEAD) && bio->bi_status != BLK_STS_INVAL;
}

139
drivers/md/raid1.c
View File

@ -604,9 +604,9 @@ static void update_read_sectors(struct r1conf *conf, int disk,
struct raid1_info *info = &conf->mirrors[disk];
atomic_inc(&info->rdev->nr_pending);
if (info->next_seq_sect != this_sector)
info->seq_start = this_sector;
info->next_seq_sect = this_sector + len;
if (READ_ONCE(info->next_seq_sect) != this_sector)
WRITE_ONCE(info->seq_start, this_sector);
WRITE_ONCE(info->next_seq_sect, this_sector + len);
}
static int choose_first_rdev(struct r1conf *conf, struct r1bio *r1_bio,
@ -735,8 +735,7 @@ static int choose_slow_rdev(struct r1conf *conf, struct r1bio *r1_bio,
static bool is_sequential(struct r1conf *conf, int disk, struct r1bio *r1_bio)
{
/* TODO: address issues with this check and concurrency. */
return conf->mirrors[disk].next_seq_sect == r1_bio->sector ||
return READ_ONCE(conf->mirrors[disk].next_seq_sect) == r1_bio->sector ||
READ_ONCE(conf->mirrors[disk].head_position) == r1_bio->sector;
}
@ -747,15 +746,18 @@ static bool is_sequential(struct r1conf *conf, int disk, struct r1bio *r1_bio)
static bool should_choose_next(struct r1conf *conf, int disk)
{
struct raid1_info *mirror = &conf->mirrors[disk];
sector_t seq_start, next_seq_sect;
int opt_iosize;
if (!test_bit(Nonrot, &mirror->rdev->flags))
return false;
opt_iosize = bdev_io_opt(mirror->rdev->bdev) >> 9;
return opt_iosize > 0 && mirror->seq_start != MaxSector &&
mirror->next_seq_sect > opt_iosize &&
mirror->next_seq_sect - opt_iosize >= mirror->seq_start;
seq_start = READ_ONCE(mirror->seq_start);
next_seq_sect = READ_ONCE(mirror->next_seq_sect);
return opt_iosize > 0 && seq_start != MaxSector &&
next_seq_sect > opt_iosize &&
next_seq_sect - opt_iosize >= seq_start;
}
static bool rdev_readable(struct md_rdev *rdev, struct r1bio *r1_bio)
@ -1051,10 +1053,8 @@ static void lower_barrier(struct r1conf *conf, sector_t sector_nr)
wake_up(&conf->wait_barrier);
}
static bool _wait_barrier(struct r1conf *conf, int idx, bool nowait)
static void _wait_barrier(struct r1conf *conf, int idx)
{
bool ret = true;
/*
* We need to increase conf->nr_pending[idx] very early here,
* then raise_barrier() can be blocked when it waits for
@ -1085,7 +1085,7 @@ static bool _wait_barrier(struct r1conf *conf, int idx, bool nowait)
*/
if (!READ_ONCE(conf->array_frozen) &&
!atomic_read(&conf->barrier[idx]))
return ret;
return;
/*
* After holding conf->resync_lock, conf->nr_pending[idx]
@ -1104,26 +1104,18 @@ static bool _wait_barrier(struct r1conf *conf, int idx, bool nowait)
wake_up_barrier(conf);
/* Wait for the barrier in same barrier unit bucket to drop. */
/* Return false when nowait flag is set */
if (nowait) {
ret = false;
} else {
wait_event_lock_irq(conf->wait_barrier,
!conf->array_frozen &&
!atomic_read(&conf->barrier[idx]),
conf->resync_lock);
atomic_inc(&conf->nr_pending[idx]);
}
wait_event_lock_irq(conf->wait_barrier, !conf->array_frozen &&
!atomic_read(&conf->barrier[idx]),
conf->resync_lock);
atomic_inc(&conf->nr_pending[idx]);
atomic_dec(&conf->nr_waiting[idx]);
spin_unlock_irq(&conf->resync_lock);
return ret;
}
static bool wait_read_barrier(struct r1conf *conf, sector_t sector_nr, bool nowait)
static void wait_read_barrier(struct r1conf *conf, sector_t sector_nr)
{
int idx = sector_to_idx(sector_nr);
bool ret = true;
/*
* Very similar to _wait_barrier(). The difference is, for read
@ -1135,7 +1127,7 @@ static bool wait_read_barrier(struct r1conf *conf, sector_t sector_nr, bool nowa
atomic_inc(&conf->nr_pending[idx]);
if (!READ_ONCE(conf->array_frozen))
return ret;
return;
spin_lock_irq(&conf->resync_lock);
atomic_inc(&conf->nr_waiting[idx]);
@ -1147,27 +1139,19 @@ static bool wait_read_barrier(struct r1conf *conf, sector_t sector_nr, bool nowa
wake_up_barrier(conf);
/* Wait for array to be unfrozen */
/* Return false when nowait flag is set */
if (nowait) {
/* Return false when nowait flag is set */
ret = false;
} else {
wait_event_lock_irq(conf->wait_barrier,
!conf->array_frozen,
conf->resync_lock);
atomic_inc(&conf->nr_pending[idx]);
}
wait_event_lock_irq(conf->wait_barrier, !conf->array_frozen,
conf->resync_lock);
atomic_inc(&conf->nr_pending[idx]);
atomic_dec(&conf->nr_waiting[idx]);
spin_unlock_irq(&conf->resync_lock);
return ret;
}
static bool wait_barrier(struct r1conf *conf, sector_t sector_nr, bool nowait)
static void wait_barrier(struct r1conf *conf, sector_t sector_nr)
{
int idx = sector_to_idx(sector_nr);
return _wait_barrier(conf, idx, nowait);
_wait_barrier(conf, idx);
}
static void _allow_barrier(struct r1conf *conf, int idx)
@ -1342,7 +1326,6 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio,
int max_sectors;
int rdisk;
bool r1bio_existed = !!r1_bio;
bool nowait = bio->bi_opf & REQ_NOWAIT;
/*
* An md cloned bio indicates we are in the error path.
@ -1362,16 +1345,7 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio,
* Still need barrier for READ in case that whole
* array is frozen.
*/
if (!wait_read_barrier(conf, bio->bi_iter.bi_sector, nowait)) {
bio_wouldblock_error(bio);
if (r1bio_existed) {
set_bit(R1BIO_Returned, &r1_bio->state);
raid_end_bio_io(r1_bio);
}
return;
}
wait_read_barrier(conf, bio->bi_iter.bi_sector);
if (!r1_bio)
r1_bio = alloc_r1bio(mddev, bio);
@ -1406,14 +1380,10 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio,
md_bitmap_enabled(mddev, false)) {
/*
* Reading from a write-mostly device must take care not to
* over-take any writes that are 'behind'
*/
mddev_add_trace_msg(mddev, "raid1 wait behind writes");
if (!mddev->bitmap_ops->wait_behind_writes(mddev, nowait)) {
bio_wouldblock_error(bio);
set_bit(R1BIO_Returned, &r1_bio->state);
goto err_handle;
}
* over-take any writes that are 'behind'
*/
mddev_add_trace_msg(mddev, "raid1 wait behind writes");
mddev->bitmap_ops->wait_behind_writes(mddev);
}
if (max_sectors < bio_sectors(bio)) {
@ -1435,7 +1405,6 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio,
}
read_bio = bio_alloc_clone(mirror->rdev->bdev, bio, gfp,
&mddev->bio_set);
read_bio->bi_opf &= ~REQ_NOWAIT;
r1_bio->bios[rdisk] = read_bio;
read_bio->bi_iter.bi_sector = r1_bio->sector +
@ -1454,7 +1423,7 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio,
raid_end_bio_io(r1_bio);
}
static bool wait_blocked_rdev(struct mddev *mddev, struct bio *bio)
static void wait_blocked_rdev(struct mddev *mddev, struct bio *bio)
{
struct r1conf *conf = mddev->private;
int disks = conf->raid_disks * 2;
@ -1474,9 +1443,6 @@ static bool wait_blocked_rdev(struct mddev *mddev, struct bio *bio)
set_bit(BlockedBadBlocks, &rdev->flags);
if (rdev_blocked(rdev)) {
if (bio->bi_opf & REQ_NOWAIT)
return false;
mddev_add_trace_msg(rdev->mddev, "raid1 wait rdev %d blocked",
rdev->raid_disk);
atomic_inc(&rdev->nr_pending);
@ -1484,8 +1450,6 @@ static bool wait_blocked_rdev(struct mddev *mddev, struct bio *bio)
goto retry;
}
}
return true;
}
static void raid1_start_write_behind(struct mddev *mddev, struct r1bio *r1_bio,
@ -1521,18 +1485,13 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio,
unsigned long flags;
int first_clone;
bool write_behind = false;
bool nowait = bio->bi_opf & REQ_NOWAIT;
bool atomic = bio->bi_opf & REQ_ATOMIC;
bool is_discard = op_is_discard(bio->bi_opf);
sector_t sector = bio->bi_iter.bi_sector;
if (mddev_is_clustered(mddev) &&
mddev->cluster_ops->area_resyncing(mddev, WRITE, sector,
bio_end_sector(bio))) {
if (nowait) {
bio_wouldblock_error(bio);
return false;
}
wait_event_idle(conf->wait_barrier,
!mddev->cluster_ops->area_resyncing(mddev, WRITE,
sector,
@ -1544,15 +1503,9 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio,
* thread has put up a bar for new requests.
* Continue immediately if no resync is active currently.
*/
if (!wait_barrier(conf, sector, nowait)) {
bio_wouldblock_error(bio);
return false;
}
wait_barrier(conf, sector);
if (!wait_blocked_rdev(mddev, bio)) {
bio_wouldblock_error(bio);
goto err_allow_barrier;
}
wait_blocked_rdev(mddev, bio);
r1_bio = alloc_r1bio(mddev, bio);
r1_bio->sectors = max_sectors;
@ -1579,6 +1532,8 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio,
*/
if (!is_discard && rdev && test_bit(WriteMostly, &rdev->flags))
write_behind = true;
if (atomic && max_sectors > BIO_MAX_VECS * (PAGE_SIZE >> 9))
write_behind = false;
r1_bio->bios[i] = NULL;
if (!rdev || test_bit(Faulty, &rdev->flags))
@ -1604,19 +1559,6 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio,
if (is_bad) {
int good_sectors;
/*
* We cannot atomically write this, so just
* error in that case. It could be possible to
* atomically write other mirrors, but the
* complexity of supporting that is not worth
* the benefit.
*/
if (bio->bi_opf & REQ_ATOMIC) {
bio->bi_status = BLK_STS_NOTSUPP;
bio_endio(bio);
goto err_dec_pending;
}
good_sectors = first_bad - sector;
if (good_sectors < max_sectors)
max_sectors = good_sectors;
@ -1637,6 +1579,11 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio,
max_sectors = min_t(int, max_sectors,
BIO_MAX_VECS * (PAGE_SIZE >> 9));
if (max_sectors < bio_sectors(bio)) {
if (atomic) {
bio_io_error(bio);
goto err_dec_pending;
}
bio = bio_submit_split_bioset(bio, max_sectors,
&conf->bio_split);
if (!bio)
@ -1681,7 +1628,6 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio,
wait_for_serialization(rdev, r1_bio);
}
mbio->bi_opf &= ~REQ_NOWAIT;
r1_bio->bios[i] = mbio;
mbio->bi_iter.bi_sector = sector + rdev->data_offset;
@ -1720,8 +1666,6 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio,
}
free_r1bio(r1_bio);
err_allow_barrier:
allow_barrier(conf, sector);
return false;
@ -1850,7 +1794,7 @@ static void close_sync(struct r1conf *conf)
int idx;
for (idx = 0; idx < BARRIER_BUCKETS_NR; idx++) {
_wait_barrier(conf, idx, false);
_wait_barrier(conf, idx);
_allow_barrier(conf, idx);
}
@ -3228,6 +3172,7 @@ static int raid1_set_limits(struct mddev *mddev)
md_init_stacking_limits(&lim);
lim.max_write_zeroes_sectors = 0;
lim.max_hw_wzeroes_unmap_sectors = 0;
lim.chunk_sectors = BARRIER_UNIT_SECTOR_SIZE;
lim.logical_block_size = mddev->logical_block_size;
lim.features |= BLK_FEAT_ATOMIC_WRITES;
lim.features |= BLK_FEAT_PCI_P2PDMA;
@ -3485,8 +3430,6 @@ static void *raid1_takeover(struct mddev *mddev)
mddev->new_chunk_sectors = 0;
conf = setup_conf(mddev);
if (!IS_ERR(conf)) {
/* Array must appear to be quiesced */
conf->array_frozen = 1;
mddev_clear_unsupported_flags(mddev,
UNSUPPORTED_MDDEV_FLAGS);
}

227
drivers/md/raid10.c
View File

@ -103,13 +103,23 @@ static inline struct r10bio *get_resync_r10bio(struct bio *bio)
return get_resync_pages(bio)->raid_bio;
}
static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data)
static inline int calc_r10bio_size(unsigned int raid_disks)
{
struct r10conf *conf = data;
int size = offsetof(struct r10bio, devs[conf->geo.raid_disks]);
return offsetof(struct r10bio, devs[raid_disks]);
}
/* allocate a r10bio with room for raid_disks entries in the
* bios array */
static mempool_t *create_r10bio_pool(unsigned int raid_disks)
{
int size = calc_r10bio_size(raid_disks);
return mempool_create_kmalloc_pool(NR_RAID_BIOS, size);
}
static struct r10bio *alloc_r10bio(unsigned int raid_disks, gfp_t gfp_flags)
{
int size = calc_r10bio_size(raid_disks);
/* allocate a r10bio sized for current geometry */
return kzalloc(size, gfp_flags);
}
@ -137,7 +147,7 @@ static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data)
int nalloc, nalloc_rp;
struct resync_pages *rps;
r10_bio = r10bio_pool_alloc(gfp_flags, conf);
r10_bio = alloc_r10bio(conf->geo.raid_disks, gfp_flags);
if (!r10_bio)
return NULL;
@ -277,7 +287,7 @@ static void free_r10bio(struct r10bio *r10_bio)
struct r10conf *conf = r10_bio->mddev->private;
put_all_bios(conf, r10_bio);
mempool_free(r10_bio, &conf->r10bio_pool);
mempool_free(r10_bio, conf->r10bio_pool);
}
static void put_buf(struct r10bio *r10_bio)
@ -321,20 +331,24 @@ static void raid_end_bio_io(struct r10bio *r10_bio)
{
struct bio *bio = r10_bio->master_bio;
struct r10conf *conf = r10_bio->mddev->private;
bool returned = true;
if (!test_and_set_bit(R10BIO_Returned, &r10_bio->state)) {
if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
bio->bi_status = BLK_STS_IOERR;
bio_endio(bio);
returned = false;
}
free_r10bio(r10_bio);
if (!returned)
bio_endio(bio);
/*
* Wake up any possible resync thread that waits for the device
* to go idle.
*/
allow_barrier(conf);
free_r10bio(r10_bio);
}
/*
@ -1002,32 +1016,22 @@ static bool wait_barrier_nolock(struct r10conf *conf)
return false;
}
static bool wait_barrier(struct r10conf *conf, bool nowait)
static void wait_barrier(struct r10conf *conf)
{
bool ret = true;
if (wait_barrier_nolock(conf))
return true;
return;
write_seqlock_irq(&conf->resync_lock);
if (conf->barrier) {
/* Return false when nowait flag is set */
if (nowait) {
ret = false;
} else {
conf->nr_waiting++;
mddev_add_trace_msg(conf->mddev, "raid10 wait barrier");
wait_event_barrier(conf, stop_waiting_barrier(conf));
conf->nr_waiting--;
}
conf->nr_waiting++;
mddev_add_trace_msg(conf->mddev, "raid10 wait barrier");
wait_event_barrier(conf, stop_waiting_barrier(conf));
conf->nr_waiting--;
if (!conf->nr_waiting)
wake_up(&conf->wait_barrier);
}
/* Only increment nr_pending when we wait */
if (ret)
atomic_inc(&conf->nr_pending);
atomic_inc(&conf->nr_pending);
write_sequnlock_irq(&conf->resync_lock);
return ret;
}
static void allow_barrier(struct r10conf *conf)
@ -1119,30 +1123,22 @@ static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule)
* currently.
* 2. If IO spans the reshape position. Need to wait for reshape to pass.
*/
static bool regular_request_wait(struct mddev *mddev, struct r10conf *conf,
static void regular_request_wait(struct mddev *mddev, struct r10conf *conf,
struct bio *bio, sector_t sectors)
{
/* Bail out if REQ_NOWAIT is set for the bio */
if (!wait_barrier(conf, bio->bi_opf & REQ_NOWAIT)) {
bio_wouldblock_error(bio);
return false;
}
wait_barrier(conf);
while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
bio->bi_iter.bi_sector < conf->reshape_progress &&
bio->bi_iter.bi_sector + sectors > conf->reshape_progress) {
allow_barrier(conf);
if (bio->bi_opf & REQ_NOWAIT) {
bio_wouldblock_error(bio);
return false;
}
mddev_add_trace_msg(conf->mddev, "raid10 wait reshape");
wait_event(conf->wait_barrier,
conf->reshape_progress <= bio->bi_iter.bi_sector ||
conf->reshape_progress >= bio->bi_iter.bi_sector +
sectors);
wait_barrier(conf, false);
wait_barrier(conf);
}
return true;
}
static void raid10_read_request(struct mddev *mddev, struct bio *bio,
@ -1191,10 +1187,7 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio,
}
}
if (!regular_request_wait(mddev, conf, bio, r10_bio->sectors)) {
free_r10bio(r10_bio);
return;
}
regular_request_wait(mddev, conf, bio, r10_bio->sectors);
rdev = read_balance(conf, r10_bio, &max_sectors);
if (!rdev) {
@ -1215,7 +1208,7 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio,
allow_barrier(conf);
bio = bio_submit_split_bioset(bio, max_sectors,
&conf->bio_split);
wait_barrier(conf, false);
wait_barrier(conf);
if (!bio) {
set_bit(R10BIO_Returned, &r10_bio->state);
goto err_handle;
@ -1231,7 +1224,6 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio,
r10_bio->master_bio = bio;
}
read_bio = bio_alloc_clone(rdev->bdev, bio, gfp, &mddev->bio_set);
read_bio->bi_opf &= ~REQ_NOWAIT;
r10_bio->devs[slot].bio = read_bio;
r10_bio->devs[slot].rdev = rdev;
@ -1265,7 +1257,6 @@ static void raid10_write_one_disk(struct mddev *mddev, struct r10bio *r10_bio,
conf->mirrors[devnum].rdev;
mbio = bio_alloc_clone(rdev->bdev, bio, GFP_NOIO, &mddev->bio_set);
mbio->bi_opf &= ~REQ_NOWAIT;
if (replacement)
r10_bio->devs[n_copy].repl_bio = mbio;
else
@ -1344,7 +1335,7 @@ static void wait_blocked_dev(struct mddev *mddev, struct r10bio *r10_bio)
"raid10 %s wait rdev %d blocked",
__func__, blocked_rdev->raid_disk);
md_wait_for_blocked_rdev(blocked_rdev, mddev);
wait_barrier(conf, false);
wait_barrier(conf);
goto retry_wait;
}
}
@ -1356,33 +1347,20 @@ static bool raid10_write_request(struct mddev *mddev, struct bio *bio,
int i, k;
sector_t sectors;
int max_sectors;
bool atomic = bio->bi_opf & REQ_ATOMIC;
if ((mddev_is_clustered(mddev) &&
mddev->cluster_ops->area_resyncing(mddev, WRITE,
bio->bi_iter.bi_sector,
bio_end_sector(bio)))) {
DEFINE_WAIT(w);
/* Bail out if REQ_NOWAIT is set for the bio */
if (bio->bi_opf & REQ_NOWAIT) {
bio_wouldblock_error(bio);
return false;
}
for (;;) {
prepare_to_wait(&conf->wait_barrier,
&w, TASK_IDLE);
if (!mddev->cluster_ops->area_resyncing(mddev, WRITE,
bio->bi_iter.bi_sector, bio_end_sector(bio)))
break;
schedule();
}
finish_wait(&conf->wait_barrier, &w);
wait_event_idle(conf->wait_barrier,
!mddev->cluster_ops->area_resyncing(mddev, WRITE,
bio->bi_iter.bi_sector,
bio_end_sector(bio)));
}
sectors = r10_bio->sectors;
if (!regular_request_wait(mddev, conf, bio, sectors)) {
free_r10bio(r10_bio);
return false;
}
regular_request_wait(mddev, conf, bio, sectors);
if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
(mddev->reshape_backwards
@ -1395,11 +1373,6 @@ static bool raid10_write_request(struct mddev *mddev, struct bio *bio,
set_mask_bits(&mddev->sb_flags, 0,
BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
md_wakeup_thread(mddev->thread);
if (bio->bi_opf & REQ_NOWAIT) {
allow_barrier(conf);
bio_wouldblock_error(bio);
return false;
}
mddev_add_trace_msg(conf->mddev,
"raid10 wait reshape metadata");
wait_event(mddev->sb_wait,
@ -1462,16 +1435,6 @@ static bool raid10_write_request(struct mddev *mddev, struct bio *bio,
if (is_bad) {
int good_sectors;
/*
* We cannot atomically write this, so just
* error in that case. It could be possible to
* atomically write other mirrors, but the
* complexity of supporting that is not worth
* the benefit.
*/
if (bio->bi_opf & REQ_ATOMIC)
goto err_handle;
good_sectors = first_bad - dev_sector;
if (good_sectors < max_sectors)
max_sectors = good_sectors;
@ -1491,10 +1454,11 @@ static bool raid10_write_request(struct mddev *mddev, struct bio *bio,
r10_bio->sectors = max_sectors;
if (r10_bio->sectors < bio_sectors(bio)) {
allow_barrier(conf);
if (atomic)
goto err_handle;
bio = bio_submit_split_bioset(bio, r10_bio->sectors,
&conf->bio_split);
wait_barrier(conf, false);
if (!bio) {
set_bit(R10BIO_Returned, &r10_bio->state);
goto err_handle;
@ -1542,7 +1506,7 @@ static bool __make_request(struct mddev *mddev, struct bio *bio, int sectors)
struct r10conf *conf = mddev->private;
struct r10bio *r10_bio;
r10_bio = mempool_alloc(&conf->r10bio_pool, GFP_NOIO);
r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
r10_bio->master_bio = bio;
r10_bio->sectors = sectors;
@ -1577,9 +1541,11 @@ static void raid_end_discard_bio(struct r10bio *r10bio)
free_r10bio(r10bio);
r10bio = first_r10bio;
} else {
struct bio *master_bio = r10bio->master_bio;
md_write_end(r10bio->mddev);
bio_endio(r10bio->master_bio);
free_r10bio(r10bio);
bio_endio(master_bio);
break;
}
}
@ -1637,11 +1603,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio)
if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
return -EAGAIN;
if (!wait_barrier(conf, bio->bi_opf & REQ_NOWAIT)) {
bio_wouldblock_error(bio);
md_write_end(mddev);
return 0;
}
wait_barrier(conf);
/*
* Check reshape again to avoid reshape happens after checking
@ -1692,7 +1654,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio)
allow_barrier(conf);
/* Resend the fist split part */
submit_bio_noacct(split);
wait_barrier(conf, false);
wait_barrier(conf);
}
div_u64_rem(bio_end, stripe_size, &remainder);
if (remainder) {
@ -1712,7 +1674,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio)
/* Resend the second split part */
submit_bio_noacct(bio);
bio = split;
wait_barrier(conf, false);
wait_barrier(conf);
}
bio_start = bio->bi_iter.bi_sector;
@ -1742,7 +1704,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio)
(last_stripe_index << geo->chunk_shift);
retry_discard:
r10_bio = mempool_alloc(&conf->r10bio_pool, GFP_NOIO);
r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
r10_bio->mddev = mddev;
r10_bio->state = 0;
r10_bio->sectors = 0;
@ -1870,7 +1832,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio)
end_disk_offset += geo->stride;
atomic_inc(&first_r10bio->remaining);
raid_end_discard_bio(r10_bio);
wait_barrier(conf, false);
wait_barrier(conf);
goto retry_discard;
}
@ -1886,6 +1848,7 @@ static bool raid10_make_request(struct mddev *mddev, struct bio *bio)
{
struct r10conf *conf = mddev->private;
sector_t chunk_mask = (conf->geo.chunk_mask & conf->prev.chunk_mask);
const int rw = bio_data_dir(bio);
int chunk_sects = chunk_mask + 1;
int sectors = bio_sectors(bio);
@ -1911,6 +1874,15 @@ static bool raid10_make_request(struct mddev *mddev, struct bio *bio)
sectors = chunk_sects -
(bio->bi_iter.bi_sector &
(chunk_sects - 1));
bio = mddev_bio_split_at_reshape_offset(mddev, bio, &sectors,
&conf->bio_split);
if (!bio) {
if (rw == WRITE)
md_write_end(mddev);
return true;
}
if (!__make_request(mddev, bio, sectors))
md_write_end(mddev);
@ -2069,7 +2041,7 @@ static void print_conf(struct r10conf *conf)
static void close_sync(struct r10conf *conf)
{
wait_barrier(conf, false);
wait_barrier(conf);
allow_barrier(conf);
mempool_exit(&conf->r10buf_pool);
@ -3365,7 +3337,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
struct md_rdev *rdev = conf->mirrors[j].rdev;
if (rdev == NULL || test_bit(Faulty, &rdev->flags)) {
still_degraded = false;
still_degraded = true;
break;
}
}
@ -3844,7 +3816,7 @@ static void raid10_free_conf(struct r10conf *conf)
if (!conf)
return;
mempool_exit(&conf->r10bio_pool);
mempool_destroy(conf->r10bio_pool);
kfree(conf->mirrors);
kfree(conf->mirrors_old);
kfree(conf->mirrors_new);
@ -3891,9 +3863,8 @@ static struct r10conf *setup_conf(struct mddev *mddev)
conf->geo = geo;
conf->copies = copies;
err = mempool_init(&conf->r10bio_pool, NR_RAID_BIOS, r10bio_pool_alloc,
rbio_pool_free, conf);
if (err)
conf->r10bio_pool = create_r10bio_pool(conf->geo.raid_disks);
if (!conf->r10bio_pool)
goto out;
err = bioset_init(&conf->bio_split, BIO_POOL_SIZE, 0, 0);
@ -4284,6 +4255,10 @@ static int raid10_check_reshape(struct mddev *mddev)
if (conf->geo.far_copies != 1 && !conf->geo.far_offset)
return -EINVAL;
if (mddev->bitmap_id == ID_LLBITMAP &&
mddev->new_chunk_sectors &&
mddev->new_chunk_sectors < mddev->chunk_sectors)
return -EOPNOTSUPP;
if (setup_geo(&geo, mddev, geo_start) != conf->copies)
/* mustn't change number of copies */
@ -4387,9 +4362,16 @@ static int raid10_start_reshape(struct mddev *mddev)
struct md_rdev *rdev;
int spares = 0;
int ret;
mempool_t *new_pool = NULL;
if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
return -EBUSY;
if (md_bitmap_enabled(mddev, false) &&
mddev->bitmap_ops->reshape_can_start) {
ret = mddev->bitmap_ops->reshape_can_start(mddev);
if (ret)
return ret;
}
if (setup_geo(&new, mddev, geo_start) != conf->copies)
return -EINVAL;
@ -4423,6 +4405,11 @@ static int raid10_start_reshape(struct mddev *mddev)
return -EINVAL;
conf->offset_diff = min_offset_diff;
if (mddev->delta_disks > 0) {
new_pool = create_r10bio_pool(new.raid_disks);
if (!new_pool)
return -ENOMEM;
}
spin_lock_irq(&conf->device_lock);
if (conf->mirrors_new) {
memcpy(conf->mirrors_new, conf->mirrors,
@ -4523,6 +4510,10 @@ static int raid10_start_reshape(struct mddev *mddev)
mddev->raid_disks = conf->geo.raid_disks;
mddev->reshape_position = conf->reshape_progress;
set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
if (new_pool) {
mempool_destroy(conf->r10bio_pool);
conf->r10bio_pool = new_pool;
}
clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
@ -4545,6 +4536,7 @@ static int raid10_start_reshape(struct mddev *mddev)
conf->reshape_safe = MaxSector;
mddev->reshape_position = MaxSector;
spin_unlock_irq(&conf->device_lock);
mempool_destroy(new_pool);
return ret;
}
@ -4702,7 +4694,14 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
if (need_flush ||
time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
/* Need to update reshape_position in metadata */
wait_barrier(conf, false);
wait_barrier(conf);
if (md_bitmap_enabled(mddev, false) &&
mddev->bitmap_ops->reshape_mark &&
conf->reshape_safe != conf->reshape_progress) {
mddev->bitmap_ops->reshape_mark(mddev, conf->reshape_safe,
conf->reshape_progress);
mddev->bitmap_ops->unplug(mddev, true);
}
mddev->reshape_position = conf->reshape_progress;
if (mddev->reshape_backwards)
mddev->curr_resync_completed = raid10_size(mddev, 0, 0)
@ -4901,9 +4900,19 @@ static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio)
static void end_reshape(struct r10conf *conf)
{
struct mddev *mddev = conf->mddev;
if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery))
return;
if (md_bitmap_enabled(mddev, false) &&
mddev->bitmap_ops->reshape_mark &&
conf->reshape_safe != conf->reshape_progress) {
mddev->bitmap_ops->reshape_mark(mddev, conf->reshape_safe,
conf->reshape_progress);
mddev->bitmap_ops->unplug(mddev, true);
}
spin_lock_irq(&conf->device_lock);
conf->prev = conf->geo;
md_finish_reshape(conf->mddev);
@ -5035,10 +5044,15 @@ static void end_reshape_request(struct r10bio *r10_bio)
static void raid10_finish_reshape(struct mddev *mddev)
{
struct r10conf *conf = mddev->private;
bool llbitmap = mddev->bitmap_id == ID_LLBITMAP &&
md_bitmap_enabled(mddev, false);
if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
return;
if (llbitmap && mddev->bitmap_ops->reshape_finish)
mddev->bitmap_ops->reshape_finish(mddev);
if (mddev->delta_disks > 0) {
if (mddev->resync_offset > mddev->resync_max_sectors) {
mddev->resync_offset = mddev->resync_max_sectors;
@ -5065,6 +5079,15 @@ static void raid10_finish_reshape(struct mddev *mddev)
mddev->reshape_backwards = 0;
}
static sector_t raid10_bitmap_sync_size(struct mddev *mddev, bool previous)
{
struct r10conf *conf = mddev->private;
if (previous)
return raid10_size(mddev, 0, 0);
return raid10_size(mddev, 0, conf->geo.raid_disks);
}
static struct md_personality raid10_personality =
{
.head = {
@ -5091,6 +5114,8 @@ static struct md_personality raid10_personality =
.start_reshape = raid10_start_reshape,
.finish_reshape = raid10_finish_reshape,
.update_reshape_pos = raid10_update_reshape_pos,
.bitmap_sync_size = raid10_bitmap_sync_size,
.bitmap_array_sectors = raid10_bitmap_sync_size,
};
static int __init raid10_init(void)

2
drivers/md/raid10.h
View File

@ -87,7 +87,7 @@ struct r10conf {
*/
wait_queue_head_t wait_barrier;
mempool_t r10bio_pool;
mempool_t *r10bio_pool;
mempool_t r10buf_pool;
struct page *tmppage;
struct bio_set bio_split;

4
drivers/md/raid5-ppl.c
View File

@ -643,8 +643,10 @@ static void ppl_do_flush(struct ppl_io_unit *io)
log->disk_flush_bitmap = 0;
for (i = flushed_disks ; i < raid_disks; i++) {
if (atomic_dec_and_test(&io->pending_flushes))
if (atomic_dec_and_test(&io->pending_flushes)) {
ppl_io_unit_finished(io);
break;
}
}
}

318
drivers/md/raid5.c
View File

@ -801,7 +801,7 @@ static bool is_inactive_blocked(struct r5conf *conf, int hash)
return true;
return (atomic_read(&conf->active_stripes) <
(conf->max_nr_stripes * 3 / 4));
(READ_ONCE(conf->max_nr_stripes) * 3 / 4));
}
struct stripe_head *raid5_get_active_stripe(struct r5conf *conf,
@ -2471,11 +2471,6 @@ static int scribble_alloc(struct raid5_percpu *percpu,
sizeof(unsigned int) * (num + 2);
void *scribble;
/*
* If here is in raid array suspend context, it is in memalloc noio
* context as well, there is no potential recursive memory reclaim
* I/Os with the GFP_KERNEL flag.
*/
scribble = kvmalloc_array(cnt, obj_size, GFP_KERNEL);
if (!scribble)
return -ENOMEM;
@ -2490,6 +2485,7 @@ static int scribble_alloc(struct raid5_percpu *percpu,
static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors)
{
unsigned long cpu;
unsigned int noio_flags;
int err = 0;
/* Never shrink. */
@ -2498,6 +2494,7 @@ static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors)
return 0;
raid5_quiesce(conf->mddev, true);
noio_flags = memalloc_noio_save();
cpus_read_lock();
for_each_present_cpu(cpu) {
@ -2511,6 +2508,7 @@ static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors)
}
cpus_read_unlock();
memalloc_noio_restore(noio_flags);
raid5_quiesce(conf->mddev, false);
if (!err) {
@ -2785,6 +2783,7 @@ static void raid5_end_read_request(struct bio * bi)
} else {
int retry = 0;
int set_bad = 0;
int max_nr_stripes = READ_ONCE(conf->max_nr_stripes);
clear_bit(R5_UPTODATE, &sh->dev[i].flags);
if (!(bi->bi_status == BLK_STS_PROTECTION))
@ -2810,13 +2809,12 @@ static void raid5_end_read_request(struct bio * bi)
mdname(conf->mddev),
(unsigned long long)s,
rdev->bdev);
} else if (atomic_read(&rdev->read_errors)
> conf->max_nr_stripes) {
} else if (atomic_read(&rdev->read_errors) > max_nr_stripes) {
if (!test_bit(Faulty, &rdev->flags)) {
pr_warn("md/raid:%s: %d read_errors > %d stripes\n",
mdname(conf->mddev),
atomic_read(&rdev->read_errors),
conf->max_nr_stripes);
max_nr_stripes);
pr_warn("md/raid:%s: Too many read errors, failing device %pg.\n",
mdname(conf->mddev), rdev->bdev);
}
@ -3553,7 +3551,7 @@ static void __add_stripe_bio(struct stripe_head *sh, struct bio *bi,
sh->dev[dd_idx].sector);
if (conf->mddev->bitmap && firstwrite && !sh->batch_head) {
sh->bm_seq = conf->seq_flush+1;
sh->bm_seq = READ_ONCE(conf->seq_flush) + 1;
set_bit(STRIPE_BIT_DELAY, &sh->state);
}
}
@ -3745,17 +3743,89 @@ handle_failed_sync(struct r5conf *conf, struct stripe_head *sh,
md_sync_error(conf->mddev);
}
/*
* handle_failed_reshape - handle failed stripes when reshape failed and
* degraded devices >= max_degraded
*
* handle following kinds of stripe:
* 1. cleanup the following kinds of destination stripe:
* - new regions past the old end of the array, zero-filled in place,
* requires no source data.
* (STRIPE_EXPANDING | STRIPE_EXPAND_READY)
* - prepare source data chunks already done, and writeout failed
* (STRIPE_EXPAND_READY)
* 2. dest stripes that need source data (STRIPE_EXPANDING, no STRIPE_HANDLE)
* - these kind of stripes sit idle in the stripe cache and are never seen
* by handle_stripe(). So clean up indirectly when their source stripe
* (type 3) is processed.
* 3. src stripes (STRIPE_EXPAND_SOURCE)
* - hit handle_stripe() after their member disks are marked Faulty.
* - clear STRIPE_EXPAND_SOURCE, finds and cleanup all dependent destination
* stripes that were waiting for data.
* - walks the source's data disks, compute the corresponding destination
* sector, looks up the destination stripe, and do cleanup(clear flags,
* dec counters, call md_done_sync())
*/
static void handle_failed_reshape(struct r5conf *conf, struct stripe_head *sh,
struct stripe_head_state *s)
{
int i;
bool was_expanding = test_and_clear_bit(STRIPE_EXPANDING, &sh->state);
bool was_ready = test_and_clear_bit(STRIPE_EXPAND_READY, &sh->state);
if (was_expanding || was_ready) {
atomic_dec(&conf->reshape_stripes);
wake_up(&conf->wait_for_reshape);
md_done_sync(conf->mddev, RAID5_STRIPE_SECTORS(conf));
}
s->expanded = 0;
s->expanding = 0;
/* release the destination stripes that are waiting to be filled */
if (test_and_clear_bit(STRIPE_EXPAND_SOURCE, &sh->state)) {
for (i = 0; i < sh->disks; i++) {
int dd_idx;
struct stripe_head *sh2;
sector_t bn, sec;
if (i == sh->pd_idx)
continue;
if (conf->level == 6 && i == sh->qd_idx)
continue;
bn = raid5_compute_blocknr(sh, i, 1);
sec = raid5_compute_sector(conf, bn, 0, &dd_idx, NULL);
sh2 = raid5_get_active_stripe(conf, NULL, sec,
R5_GAS_NOBLOCK |
R5_GAS_NOQUIESCE);
if (!sh2)
continue;
if (test_and_clear_bit(STRIPE_EXPANDING, &sh2->state)) {
atomic_dec(&conf->reshape_stripes);
wake_up(&conf->wait_for_reshape);
md_done_sync(conf->mddev,
RAID5_STRIPE_SECTORS(conf));
}
clear_bit(STRIPE_EXPAND_READY, &sh2->state);
raid5_release_stripe(sh2);
}
}
}
static int want_replace(struct stripe_head *sh, int disk_idx)
{
struct md_rdev *rdev;
int rv = 0;
rdev = sh->raid_conf->disks[disk_idx].replacement;
if (rdev
&& !test_bit(Faulty, &rdev->flags)
&& !test_bit(In_sync, &rdev->flags)
&& (rdev->recovery_offset <= sh->sector
|| rdev->mddev->resync_offset <= sh->sector))
if (rdev && !test_bit(Faulty, &rdev->flags) &&
!test_bit(In_sync, &rdev->flags) &&
(READ_ONCE(rdev->recovery_offset) <= sh->sector ||
rdev->mddev->resync_offset <= sh->sector))
rv = 1;
return rv;
}
@ -4672,7 +4742,8 @@ static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
*/
rdev = conf->disks[i].replacement;
if (rdev && !test_bit(Faulty, &rdev->flags) &&
rdev->recovery_offset >= sh->sector + RAID5_STRIPE_SECTORS(conf) &&
READ_ONCE(rdev->recovery_offset) >=
sh->sector + RAID5_STRIPE_SECTORS(conf) &&
!rdev_has_badblock(rdev, sh->sector,
RAID5_STRIPE_SECTORS(conf)))
set_bit(R5_ReadRepl, &dev->flags);
@ -4714,7 +4785,7 @@ static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
} else if (test_bit(In_sync, &rdev->flags))
set_bit(R5_Insync, &dev->flags);
else if (sh->sector + RAID5_STRIPE_SECTORS(conf) <=
rdev->recovery_offset) {
READ_ONCE(rdev->recovery_offset)) {
/*
* in sync if:
* - normal IO, or
@ -5025,6 +5096,8 @@ static void handle_stripe(struct stripe_head *sh)
handle_failed_stripe(conf, sh, &s, disks);
if (s.syncing + s.replacing)
handle_failed_sync(conf, sh, &s);
if (s.expanding + s.expanded)
handle_failed_reshape(conf, sh, &s);
}
/* Now we check to see if any write operations have recently
@ -5458,13 +5531,13 @@ static int raid5_read_one_chunk(struct mddev *mddev, struct bio *raid_bio)
rdev = conf->disks[dd_idx].replacement;
if (!rdev || test_bit(Faulty, &rdev->flags) ||
rdev->recovery_offset < end_sector) {
READ_ONCE(rdev->recovery_offset) < end_sector) {
rdev = conf->disks[dd_idx].rdev;
if (!rdev)
return 0;
if (test_bit(Faulty, &rdev->flags) ||
!(test_bit(In_sync, &rdev->flags) ||
rdev->recovery_offset >= end_sector))
READ_ONCE(rdev->recovery_offset) >= end_sector))
return 0;
}
@ -5718,13 +5791,8 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi)
struct bio *orig_bi = bi;
int stripe_sectors;
/* We need to handle this when io_uring supports discard/trim */
if (WARN_ON_ONCE(bi->bi_opf & REQ_NOWAIT))
return;
if (mddev->reshape_position != MaxSector)
/* Skip discard while reshape is happening */
return;
goto complete_again;
if (!raid5_discard_limits(mddev, bi))
return;
@ -5799,7 +5867,7 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi)
}
spin_unlock_irq(&sh->stripe_lock);
if (conf->mddev->bitmap) {
sh->bm_seq = conf->seq_flush + 1;
sh->bm_seq = READ_ONCE(conf->seq_flush) + 1;
set_bit(STRIPE_BIT_DELAY, &sh->state);
}
@ -5811,6 +5879,11 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi)
}
bio_endio(bi);
return;
complete_again:
/* Skip discard while reshape is happening */
bio_endio_status(bi, BLK_STS_AGAIN);
}
static bool ahead_of_reshape(struct mddev *mddev, sector_t sector,
@ -5942,6 +6015,34 @@ static enum reshape_loc get_reshape_loc(struct mddev *mddev,
return LOC_BEHIND_RESHAPE;
}
static void raid5_bitmap_sector_map(struct mddev *mddev, sector_t *offset,
unsigned long *sectors,
bool previous)
{
struct r5conf *conf = mddev->private;
sector_t start = *offset;
sector_t end = start + *sectors;
int sectors_per_chunk;
int dd_idx;
if (previous)
sectors_per_chunk = conf->prev_chunk_sectors *
(conf->previous_raid_disks - conf->max_degraded);
else
sectors_per_chunk = conf->chunk_sectors *
(conf->raid_disks - conf->max_degraded);
sector_div(start, sectors_per_chunk);
start *= sectors_per_chunk;
if (sector_div(end, sectors_per_chunk))
end++;
end *= sectors_per_chunk;
start = raid5_compute_sector(conf, start, previous, &dd_idx, NULL);
end = raid5_compute_sector(conf, end, previous, &dd_idx, NULL);
*offset = start;
*sectors = end - start;
}
static void raid5_bitmap_sector(struct mddev *mddev, sector_t *offset,
unsigned long *sectors)
{
@ -5949,18 +6050,11 @@ static void raid5_bitmap_sector(struct mddev *mddev, sector_t *offset,
sector_t start = *offset;
sector_t end = start + *sectors;
sector_t prev_start = start;
sector_t prev_end = end;
int sectors_per_chunk;
unsigned long prev_sectors = end - start;
enum reshape_loc loc;
int dd_idx;
sectors_per_chunk = conf->chunk_sectors *
(conf->raid_disks - conf->max_degraded);
start = round_down(start, sectors_per_chunk);
end = round_up(end, sectors_per_chunk);
start = raid5_compute_sector(conf, start, 0, &dd_idx, NULL);
end = raid5_compute_sector(conf, end, 0, &dd_idx, NULL);
raid5_bitmap_sector_map(mddev, &start, sectors, false);
end = start + *sectors;
/*
* For LOC_INSIDE_RESHAPE, this IO will wait for reshape to make
@ -5969,17 +6063,10 @@ static void raid5_bitmap_sector(struct mddev *mddev, sector_t *offset,
loc = get_reshape_loc(mddev, conf, prev_start);
if (likely(loc != LOC_AHEAD_OF_RESHAPE)) {
*offset = start;
*sectors = end - start;
return;
}
sectors_per_chunk = conf->prev_chunk_sectors *
(conf->previous_raid_disks - conf->max_degraded);
prev_start = round_down(prev_start, sectors_per_chunk);
prev_end = round_down(prev_end, sectors_per_chunk);
prev_start = raid5_compute_sector(conf, prev_start, 1, &dd_idx, NULL);
prev_end = raid5_compute_sector(conf, prev_end, 1, &dd_idx, NULL);
raid5_bitmap_sector_map(mddev, &prev_start, &prev_sectors, true);
/*
* for LOC_AHEAD_OF_RESHAPE, reshape can make progress before this IO
@ -5987,7 +6074,7 @@ static void raid5_bitmap_sector(struct mddev *mddev, sector_t *offset,
* we set bits for both.
*/
*offset = min(start, prev_start);
*sectors = max(end, prev_end) - *offset;
*sectors = max(end, prev_start + prev_sectors) - *offset;
}
static enum stripe_result make_stripe_request(struct mddev *mddev,
@ -6134,9 +6221,11 @@ static bool raid5_make_request(struct mddev *mddev, struct bio * bi)
struct r5conf *conf = mddev->private;
const int rw = bio_data_dir(bi);
struct stripe_request_ctx *ctx;
struct bio *front_bio;
sector_t logical_sector;
enum stripe_result res;
int s, stripe_cnt;
bool split = false;
bool on_wq;
if (unlikely(bi->bi_opf & REQ_PREFLUSH)) {
@ -6170,6 +6259,18 @@ static bool raid5_make_request(struct mddev *mddev, struct bio * bi)
return true;
}
front_bio = bi;
bi = mddev_bio_split_at_reshape_offset(mddev, bi, NULL,
&conf->bio_split);
if (!bi) {
if (rw == WRITE)
md_write_end(mddev);
return true;
}
if (bi != front_bio)
split = true;
front_bio = bi;
logical_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);
bi->bi_next = NULL;
@ -6191,15 +6292,6 @@ static bool raid5_make_request(struct mddev *mddev, struct bio * bi)
pr_debug("raid456: %s, logical %llu to %llu\n", __func__,
bi->bi_iter.bi_sector, ctx->last_sector);
/* Bail out if conflicts with reshape and REQ_NOWAIT is set */
if ((bi->bi_opf & REQ_NOWAIT) &&
get_reshape_loc(mddev, conf, logical_sector) == LOC_INSIDE_RESHAPE) {
bio_wouldblock_error(bi);
if (rw == WRITE)
md_write_end(mddev);
mempool_free(ctx, conf->ctx_pool);
return true;
}
md_account_bio(mddev, &bi);
/*
@ -6270,6 +6362,11 @@ static bool raid5_make_request(struct mddev *mddev, struct bio * bi)
bio_endio(bi);
wait_for_completion(&done);
front_bio->bi_status = BLK_STS_OK;
if (split) {
submit_bio_noacct(front_bio);
return true;
}
return false;
}
@ -6419,6 +6516,13 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *sk
|| test_bit(MD_RECOVERY_INTR, &mddev->recovery));
if (atomic_read(&conf->reshape_stripes) != 0)
return 0;
if (md_bitmap_enabled(mddev, false) &&
mddev->bitmap_ops->reshape_mark &&
conf->reshape_safe != conf->reshape_progress) {
mddev->bitmap_ops->reshape_mark(mddev, conf->reshape_safe,
conf->reshape_progress);
mddev->bitmap_ops->unplug(mddev, true);
}
mddev->reshape_position = conf->reshape_progress;
mddev->curr_resync_completed = sector_nr;
if (!mddev->reshape_backwards)
@ -6427,8 +6531,8 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *sk
if (rdev->raid_disk >= 0 &&
!test_bit(Journal, &rdev->flags) &&
!test_bit(In_sync, &rdev->flags) &&
rdev->recovery_offset < sector_nr)
rdev->recovery_offset = sector_nr;
READ_ONCE(rdev->recovery_offset) < sector_nr)
WRITE_ONCE(rdev->recovery_offset, sector_nr);
conf->reshape_checkpoint = jiffies;
set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
@ -6528,6 +6632,13 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *sk
|| test_bit(MD_RECOVERY_INTR, &mddev->recovery));
if (atomic_read(&conf->reshape_stripes) != 0)
goto ret;
if (md_bitmap_enabled(mddev, false) &&
mddev->bitmap_ops->reshape_mark &&
conf->reshape_safe != conf->reshape_progress) {
mddev->bitmap_ops->reshape_mark(mddev, conf->reshape_safe,
conf->reshape_progress);
mddev->bitmap_ops->unplug(mddev, true);
}
mddev->reshape_position = conf->reshape_progress;
mddev->curr_resync_completed = sector_nr;
if (!mddev->reshape_backwards)
@ -6536,8 +6647,8 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *sk
if (rdev->raid_disk >= 0 &&
!test_bit(Journal, &rdev->flags) &&
!test_bit(In_sync, &rdev->flags) &&
rdev->recovery_offset < sector_nr)
rdev->recovery_offset = sector_nr;
READ_ONCE(rdev->recovery_offset) < sector_nr)
WRITE_ONCE(rdev->recovery_offset, sector_nr);
conf->reshape_checkpoint = jiffies;
set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
md_wakeup_thread(mddev->thread);
@ -6849,12 +6960,14 @@ static void raid5d(struct md_thread *thread)
if (
!list_empty(&conf->bitmap_list)) {
/* Now is a good time to flush some bitmap updates */
conf->seq_flush++;
int seq = conf->seq_flush + 1;
WRITE_ONCE(conf->seq_flush, seq);
spin_unlock_irq(&conf->device_lock);
if (md_bitmap_enabled(mddev, true))
mddev->bitmap_ops->unplug(mddev, true);
spin_lock_irq(&conf->device_lock);
conf->seq_write = conf->seq_flush;
conf->seq_write = seq;
activate_bit_delay(conf, conf->temp_inactive_list);
}
raid5_activate_delayed(conf);
@ -7039,6 +7152,7 @@ raid5_store_stripe_size(struct mddev *mddev, const char *page, size_t len)
{
struct r5conf *conf;
unsigned long new;
unsigned int noio_flags = 0;
int err;
int size;
@ -7079,6 +7193,7 @@ raid5_store_stripe_size(struct mddev *mddev, const char *page, size_t len)
goto out_unlock;
}
noio_flags = memalloc_noio_save();
mutex_lock(&conf->cache_size_mutex);
size = conf->max_nr_stripes;
@ -7095,6 +7210,7 @@ raid5_store_stripe_size(struct mddev *mddev, const char *page, size_t len)
mutex_unlock(&conf->cache_size_mutex);
out_unlock:
memalloc_noio_restore(noio_flags);
mddev_unlock_and_resume(mddev);
return err ?: len;
}
@ -8056,9 +8172,9 @@ static int raid5_run(struct mddev *mddev)
/* Hack because v0.91 doesn't store recovery_offset properly. */
if (mddev->major_version == 0 &&
mddev->minor_version > 90)
rdev->recovery_offset = reshape_offset;
WRITE_ONCE(rdev->recovery_offset, reshape_offset);
if (rdev->recovery_offset < reshape_offset) {
if (READ_ONCE(rdev->recovery_offset) < reshape_offset) {
/* We need to check old and new layout */
if (!only_parity(rdev->raid_disk,
conf->algorithm,
@ -8213,10 +8329,10 @@ static int raid5_spare_active(struct mddev *mddev)
for (i = 0; i < conf->raid_disks; i++) {
rdev = conf->disks[i].rdev;
replacement = conf->disks[i].replacement;
if (replacement
&& replacement->recovery_offset == MaxSector
&& !test_bit(Faulty, &replacement->flags)
&& !test_and_set_bit(In_sync, &replacement->flags)) {
if (replacement &&
READ_ONCE(replacement->recovery_offset) == MaxSector &&
!test_bit(Faulty, &replacement->flags) &&
!test_and_set_bit(In_sync, &replacement->flags)) {
/* Replacement has just become active. */
if (!rdev
|| !test_and_clear_bit(In_sync, &rdev->flags))
@ -8231,10 +8347,10 @@ static int raid5_spare_active(struct mddev *mddev)
rdev->sysfs_state);
}
sysfs_notify_dirent_safe(replacement->sysfs_state);
} else if (rdev
&& rdev->recovery_offset == MaxSector
&& !test_bit(Faulty, &rdev->flags)
&& !test_and_set_bit(In_sync, &rdev->flags)) {
} else if (rdev &&
READ_ONCE(rdev->recovery_offset) == MaxSector &&
!test_bit(Faulty, &rdev->flags) &&
!test_and_set_bit(In_sync, &rdev->flags)) {
count++;
sysfs_notify_dirent_safe(rdev->sysfs_state);
}
@ -8479,6 +8595,8 @@ static int check_reshape(struct mddev *mddev)
return 0; /* nothing to do */
if (has_failed(conf))
return -EINVAL;
if (!mddev->new_chunk_sectors)
return -EINVAL;
if (mddev->delta_disks < 0 && mddev->reshape_position == MaxSector) {
/* We might be able to shrink, but the devices must
* be made bigger first.
@ -8495,6 +8613,9 @@ static int check_reshape(struct mddev *mddev)
if (!check_stripe_cache(mddev))
return -ENOSPC;
if (mddev->bitmap_id == ID_LLBITMAP &&
mddev->new_chunk_sectors < mddev->chunk_sectors)
return -EOPNOTSUPP;
if (mddev->new_chunk_sectors > mddev->chunk_sectors ||
mddev->delta_disks > 0)
if (resize_chunks(conf,
@ -8522,6 +8643,9 @@ static int raid5_start_reshape(struct mddev *mddev)
if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
return -EBUSY;
if (!mddev->new_chunk_sectors)
return -EINVAL;
if (!check_stripe_cache(mddev))
return -ENOSPC;
@ -8557,6 +8681,12 @@ static int raid5_start_reshape(struct mddev *mddev)
mdname(mddev));
return -EINVAL;
}
if (md_bitmap_enabled(mddev, false) &&
mddev->bitmap_id == ID_LLBITMAP) {
i = mddev->bitmap_ops->resize(mddev, mddev->dev_sectors, 0);
if (i)
return i;
}
atomic_set(&conf->reshape_stripes, 0);
spin_lock_irq(&conf->device_lock);
@ -8603,7 +8733,7 @@ static int raid5_start_reshape(struct mddev *mddev)
>= conf->previous_raid_disks)
set_bit(In_sync, &rdev->flags);
else
rdev->recovery_offset = 0;
WRITE_ONCE(rdev->recovery_offset, 0);
/* Failure here is OK */
sysfs_link_rdev(mddev, rdev);
@ -8641,10 +8771,19 @@ static int raid5_start_reshape(struct mddev *mddev)
*/
static void end_reshape(struct r5conf *conf)
{
struct mddev *mddev = conf->mddev;
if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) {
struct md_rdev *rdev;
if (md_bitmap_enabled(mddev, false) &&
mddev->bitmap_ops->reshape_mark &&
conf->reshape_safe != conf->reshape_progress) {
mddev->bitmap_ops->reshape_mark(mddev, conf->reshape_safe,
conf->reshape_progress);
mddev->bitmap_ops->unplug(mddev, true);
}
spin_lock_irq(&conf->device_lock);
conf->previous_raid_disks = conf->raid_disks;
md_finish_reshape(conf->mddev);
@ -8655,7 +8794,7 @@ static void end_reshape(struct r5conf *conf)
if (rdev->raid_disk >= 0 &&
!test_bit(Journal, &rdev->flags) &&
!test_bit(In_sync, &rdev->flags))
rdev->recovery_offset = MaxSector;
WRITE_ONCE(rdev->recovery_offset, MaxSector);
spin_unlock_irq(&conf->device_lock);
wake_up(&conf->wait_for_reshape);
@ -8671,8 +8810,16 @@ static void raid5_finish_reshape(struct mddev *mddev)
{
struct r5conf *conf = mddev->private;
struct md_rdev *rdev;
bool llbitmap = mddev->bitmap_id == ID_LLBITMAP &&
md_bitmap_enabled(mddev, false);
if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
if (llbitmap && mddev->bitmap_ops->reshape_finish)
mddev->bitmap_ops->reshape_finish(mddev);
if (llbitmap) {
mddev->resync_offset = 0;
mddev->resync_max_sectors = mddev->dev_sectors;
}
if (mddev->delta_disks <= 0) {
int d;
@ -8975,6 +9122,7 @@ static void *raid6_takeover(struct mddev *mddev)
static int raid5_change_consistency_policy(struct mddev *mddev, const char *buf)
{
struct r5conf *conf;
unsigned int noio_flags;
int err;
err = mddev_suspend_and_lock(mddev);
@ -8986,6 +9134,7 @@ static int raid5_change_consistency_policy(struct mddev *mddev, const char *buf)
return -ENODEV;
}
noio_flags = memalloc_noio_save();
if (strncmp(buf, "ppl", 3) == 0) {
/* ppl only works with RAID 5 */
if (!raid5_has_ppl(conf) && conf->level == 5) {
@ -9025,6 +9174,7 @@ static int raid5_change_consistency_policy(struct mddev *mddev, const char *buf)
if (!err)
md_update_sb(mddev, 1);
memalloc_noio_restore(noio_flags);
mddev_unlock_and_resume(mddev);
return err;
@ -9049,6 +9199,21 @@ static void raid5_prepare_suspend(struct mddev *mddev)
wake_up(&conf->wait_for_reshape);
}
static sector_t raid5_bitmap_sync_size(struct mddev *mddev, bool previous)
{
return mddev->dev_sectors;
}
static sector_t raid5_bitmap_array_sectors(struct mddev *mddev, bool previous)
{
struct r5conf *conf = mddev->private;
if (previous)
return raid5_size(mddev, mddev->dev_sectors,
conf->previous_raid_disks);
return raid5_size(mddev, mddev->dev_sectors, conf->raid_disks);
}
static struct md_personality raid6_personality =
{
.head = {
@ -9078,6 +9243,9 @@ static struct md_personality raid6_personality =
.change_consistency_policy = raid5_change_consistency_policy,
.prepare_suspend = raid5_prepare_suspend,
.bitmap_sector = raid5_bitmap_sector,
.bitmap_sector_map = raid5_bitmap_sector_map,
.bitmap_sync_size = raid5_bitmap_sync_size,
.bitmap_array_sectors = raid5_bitmap_array_sectors,
};
static struct md_personality raid5_personality =
{
@ -9108,6 +9276,9 @@ static struct md_personality raid5_personality =
.change_consistency_policy = raid5_change_consistency_policy,
.prepare_suspend = raid5_prepare_suspend,
.bitmap_sector = raid5_bitmap_sector,
.bitmap_sector_map = raid5_bitmap_sector_map,
.bitmap_sync_size = raid5_bitmap_sync_size,
.bitmap_array_sectors = raid5_bitmap_array_sectors,
};
static struct md_personality raid4_personality =
@ -9139,6 +9310,9 @@ static struct md_personality raid4_personality =
.change_consistency_policy = raid5_change_consistency_policy,
.prepare_suspend = raid5_prepare_suspend,
.bitmap_sector = raid5_bitmap_sector,
.bitmap_sector_map = raid5_bitmap_sector_map,
.bitmap_sync_size = raid5_bitmap_sync_size,
.bitmap_array_sectors = raid5_bitmap_array_sectors,
};
static int __init raid5_init(void)