diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c index 0f02e2956398..b8325cb09a37 100644 --- a/drivers/md/md-bitmap.c +++ b/drivers/md/md-bitmap.c @@ -1730,6 +1730,13 @@ static void bitmap_start_write(struct mddev *mddev, sector_t offset, } } +static void bitmap_prepare_range(struct mddev *mddev, sector_t *offset, + unsigned long *sectors, bool discard) +{ + if (mddev->pers->bitmap_sector) + mddev->pers->bitmap_sector(mddev, offset, sectors); +} + static void bitmap_end_write(struct mddev *mddev, sector_t offset, unsigned long sectors) { @@ -2064,23 +2071,18 @@ static void bitmap_end_behind_write(struct mddev *mddev) bitmap->mddev->bitmap_info.max_write_behind); } -static bool bitmap_wait_behind_writes(struct mddev *mddev, bool nowait) +static void bitmap_wait_behind_writes(struct mddev *mddev) { struct bitmap *bitmap = mddev->bitmap; /* wait for behind writes to complete */ if (bitmap && atomic_read(&bitmap->behind_writes) > 0) { - if (nowait) - return false; - pr_debug("md:%s: behind writes in progress - waiting to stop.\n", mdname(mddev)); /* need to kick something here to make sure I/O goes? */ wait_event(bitmap->behind_wait, atomic_read(&bitmap->behind_writes) == 0); } - - return true; } static void bitmap_destroy(struct mddev *mddev) @@ -2090,7 +2092,7 @@ static void bitmap_destroy(struct mddev *mddev) if (!bitmap) /* there was no bitmap */ return; - bitmap_wait_behind_writes(mddev, false); + bitmap_wait_behind_writes(mddev); if (!test_bit(MD_SERIALIZE_POLICY, &mddev->flags)) mddev_destroy_serial_pool(mddev, NULL); @@ -2629,10 +2631,12 @@ static ssize_t location_store(struct mddev *mddev, const char *buf, size_t len) { int rv; + unsigned int noio_flags; rv = mddev_suspend_and_lock(mddev); if (rv) return rv; + noio_flags = memalloc_noio_save(); if (mddev->pers) { if (mddev->recovery || mddev->sync_thread) { @@ -2719,6 +2723,7 @@ location_store(struct mddev *mddev, const char *buf, size_t len) } rv = 0; out: + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); if (rv) return rv; @@ -2862,7 +2867,7 @@ backlog_store(struct mddev *mddev, const char *buf, size_t len) if (!has_write_mostly) { pr_warn_ratelimited("%s: can't set backlog, no write mostly device available\n", mdname(mddev)); - mddev_unlock(mddev); + mddev_unlock_and_resume(mddev); return -EINVAL; } @@ -3083,6 +3088,7 @@ static struct bitmap_operations bitmap_ops = { .flush = bitmap_flush, .write_all = bitmap_write_all, .dirty_bits = bitmap_dirty_bits, + .prepare_range = bitmap_prepare_range, .unplug = bitmap_unplug, .daemon_work = bitmap_daemon_work, diff --git a/drivers/md/md-bitmap.h b/drivers/md/md-bitmap.h index f46674bdfeb9..b69c78174f02 100644 --- a/drivers/md/md-bitmap.h +++ b/drivers/md/md-bitmap.h @@ -29,6 +29,7 @@ enum bitmap_state { BITMAP_FIRST_USE = 3, /* llbitmap is just created */ BITMAP_CLEAN = 4, /* llbitmap is created with assume_clean */ BITMAP_DAEMON_BUSY = 5, /* llbitmap daemon is not finished after daemon_sleep */ + BITMAP_SHUTDOWN = 6, /* llbitmap is being destroyed */ BITMAP_HOSTENDIAN =15, }; @@ -93,12 +94,21 @@ struct bitmap_operations { void (*write_all)(struct mddev *mddev); void (*dirty_bits)(struct mddev *mddev, unsigned long s, unsigned long e); + /* Prepare a range for this bitmap implementation. */ + void (*prepare_range)(struct mddev *mddev, + sector_t *offset, + unsigned long *sectors, + bool discard); + void (*reshape_finish)(struct mddev *mddev); + int (*reshape_can_start)(struct mddev *mddev); + void (*reshape_mark)(struct mddev *mddev, sector_t old_pos, + sector_t new_pos); void (*unplug)(struct mddev *mddev, bool sync); void (*daemon_work)(struct mddev *mddev); void (*start_behind_write)(struct mddev *mddev); void (*end_behind_write)(struct mddev *mddev); - bool (*wait_behind_writes)(struct mddev *mddev, bool nowait); + void (*wait_behind_writes)(struct mddev *mddev); md_bitmap_fn *start_write; md_bitmap_fn *end_write; diff --git a/drivers/md/md-linear.c b/drivers/md/md-linear.c index fdff250d0d51..73b367b61b87 100644 --- a/drivers/md/md-linear.c +++ b/drivers/md/md-linear.c @@ -71,6 +71,7 @@ static int linear_set_limits(struct mddev *mddev) int err; md_init_stacking_limits(&lim); + lim.features |= BLK_FEAT_NOWAIT; lim.max_hw_sectors = mddev->chunk_sectors; lim.logical_block_size = mddev->logical_block_size; lim.max_write_zeroes_sectors = mddev->chunk_sectors; diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index 5a4e2abaa757..e1a783ee2032 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -287,6 +288,11 @@ struct llbitmap { unsigned long chunksize; /* total number of chunks */ unsigned long chunks; + /* total number of sectors tracked by current bitmap geometry */ + sector_t sync_size; + unsigned long reshape_chunksize; + unsigned long reshape_chunks; + sector_t reshape_sync_size; unsigned long last_end_sync; /* * time in seconds that dirty bits will be cleared if the page is not @@ -296,6 +302,11 @@ struct llbitmap { /* fires on first BitDirty state */ struct timer_list pending_timer; struct work_struct daemon_work; + /* + * Serialize reshape checkpoint remapping against normal I/O bitmap + * updates without blocking concurrent I/O updates on each other. + */ + rwlock_t reshape_lock; unsigned long flags; __u64 events_cleared; @@ -414,6 +425,140 @@ static char state_machine[BitStateCount][BitmapActionCount] = { }; static void __llbitmap_flush(struct mddev *mddev); +static void llbitmap_flush(struct mddev *mddev); +static void llbitmap_update_sb(void *data); + +static void llbitmap_calculate_chunks(struct mddev *mddev, sector_t blocks, + unsigned long *chunksize, + unsigned long *chunks) +{ + *chunks = DIV_ROUND_UP_SECTOR_T(blocks, *chunksize); + while (*chunks > mddev->bitmap_info.space << SECTOR_SHIFT) { + *chunksize = *chunksize << 1; + *chunks = DIV_ROUND_UP_SECTOR_T(blocks, *chunksize); + } +} + +static sector_t llbitmap_personality_sync_size(struct llbitmap *llbitmap, + bool previous) +{ + struct mddev *mddev = llbitmap->mddev; + + if (READ_ONCE(mddev->reshape_position) == MaxSector || + !mddev->private || !mddev->pers || + !mddev->pers->bitmap_sync_size) + return llbitmap->sync_size; + return mddev->pers->bitmap_sync_size(mddev, previous); +} + +static sector_t llbitmap_logical_size(struct llbitmap *llbitmap, bool previous) +{ + struct mddev *mddev = llbitmap->mddev; + + if (!mddev->private || !mddev->pers || + !mddev->pers->bitmap_array_sectors) + return llbitmap_personality_sync_size(llbitmap, previous); + return mddev->pers->bitmap_array_sectors(mddev, previous); +} + +static void llbitmap_refresh_reshape(struct llbitmap *llbitmap) +{ + unsigned long old_chunks = DIV_ROUND_UP_SECTOR_T(llbitmap->sync_size, + llbitmap->chunksize); + sector_t blocks = llbitmap_personality_sync_size(llbitmap, false); + unsigned long chunksize = llbitmap->chunksize; + unsigned long chunks = DIV_ROUND_UP_SECTOR_T(blocks, chunksize); + + llbitmap->reshape_sync_size = blocks; + llbitmap->reshape_chunksize = chunksize; + llbitmap->reshape_chunks = chunks; + llbitmap_calculate_chunks(llbitmap->mddev, blocks, + &llbitmap->reshape_chunksize, + &llbitmap->reshape_chunks); + llbitmap->chunks = max(old_chunks, llbitmap->reshape_chunks); +} + +static void llbitmap_map_layout(struct llbitmap *llbitmap, sector_t *offset, + unsigned long *sectors, bool previous) +{ + sector_t limit = llbitmap_logical_size(llbitmap, previous); + sector_t start = *offset; + sector_t end = start + *sectors; + + if (start >= limit) { + *sectors = 0; + return; + } + if (end > limit) + end = limit; + + *offset = start; + *sectors = end - start; + if (!*sectors) + return; + + if (llbitmap->mddev->pers->bitmap_sector_map) + llbitmap->mddev->pers->bitmap_sector_map(llbitmap->mddev, offset, + sectors, previous); + else if (!previous && llbitmap->mddev->pers->bitmap_sector) + llbitmap->mddev->pers->bitmap_sector(llbitmap->mddev, offset, + sectors); + + limit = llbitmap_personality_sync_size(llbitmap, previous); + start = *offset; + end = start + *sectors; + if (start >= limit) + *sectors = 0; + else if (end > limit) + *sectors = limit - start; +} + +static void llbitmap_encode_range(struct llbitmap *llbitmap, sector_t *offset, + unsigned long *sectors, bool previous) +{ + unsigned long chunksize = previous ? llbitmap->chunksize : + llbitmap->reshape_chunksize; + u64 start; + u64 end; + + if (!*sectors) { + *offset = 0; + return; + } + + start = div64_u64(*offset, chunksize); + end = div64_u64(*offset + *sectors - 1, chunksize); + *offset = (sector_t)start << llbitmap->chunkshift; + *sectors = (end - start + 1) << llbitmap->chunkshift; +} + +static void llbitmap_encode_discard_range(struct llbitmap *llbitmap, + sector_t *offset, + unsigned long *sectors, + bool previous) +{ + unsigned long chunksize = previous ? llbitmap->chunksize : + llbitmap->reshape_chunksize; + sector_t end = *offset + *sectors; + u64 start; + u64 last; + + if (!*sectors) { + *offset = 0; + return; + } + + start = DIV_ROUND_UP_SECTOR_T(*offset, chunksize); + last = div64_u64(end, chunksize); + if (start >= last) { + *offset = 0; + *sectors = 0; + return; + } + + *offset = (sector_t)start << llbitmap->chunkshift; + *sectors = (last - start) << llbitmap->chunkshift; +} static enum llbitmap_state llbitmap_read(struct llbitmap *llbitmap, loff_t pos) { @@ -510,20 +655,28 @@ static void llbitmap_write(struct llbitmap *llbitmap, enum llbitmap_state state, llbitmap_set_page_dirty(llbitmap, idx, bit, false); } +static unsigned int llbitmap_used_pages(struct llbitmap *llbitmap, + unsigned long chunks) +{ + return DIV_ROUND_UP(chunks + BITMAP_DATA_OFFSET, PAGE_SIZE); +} + static struct page *llbitmap_read_page(struct llbitmap *llbitmap, int idx) { struct mddev *mddev = llbitmap->mddev; struct page *page = NULL; struct md_rdev *rdev; - if (llbitmap->pctl && llbitmap->pctl[idx]) + if (llbitmap->pctl && idx < llbitmap->nr_pages && llbitmap->pctl[idx]) page = llbitmap->pctl[idx]->page; if (page) return page; - page = alloc_page(GFP_KERNEL | __GFP_ZERO); + page = alloc_page(GFP_NOIO | __GFP_ZERO); if (!page) return ERR_PTR(-ENOMEM); + if (idx >= llbitmap_used_pages(llbitmap, llbitmap->chunks)) + return page; rdev_for_each(rdev, mddev) { sector_t sector; @@ -594,61 +747,120 @@ static void llbitmap_free_pages(struct llbitmap *llbitmap) for (i = 0; i < llbitmap->nr_pages; i++) { struct llbitmap_page_ctl *pctl = llbitmap->pctl[i]; - if (!pctl || !pctl->page) - break; - - __free_page(pctl->page); + if (!pctl) + continue; + if (pctl->page) + __free_page(pctl->page); percpu_ref_exit(&pctl->active); + kfree(pctl); } - kfree(llbitmap->pctl[0]); kfree(llbitmap->pctl); llbitmap->pctl = NULL; } -static int llbitmap_cache_pages(struct llbitmap *llbitmap) +static struct llbitmap_page_ctl * +llbitmap_alloc_page_ctl(struct llbitmap *llbitmap, int idx) { struct llbitmap_page_ctl *pctl; - unsigned int nr_pages = DIV_ROUND_UP(llbitmap->chunks + - BITMAP_DATA_OFFSET, PAGE_SIZE); + struct page *page; unsigned int size = struct_size(pctl, dirty, BITS_TO_LONGS( llbitmap->blocks_per_page)); + + size = round_up(size, cache_line_size()); + pctl = kzalloc(size, GFP_NOIO); + if (!pctl) + return ERR_PTR(-ENOMEM); + + page = llbitmap_read_page(llbitmap, idx); + + if (IS_ERR(page)) { + kfree(pctl); + return ERR_CAST(page); + } + + if (percpu_ref_init(&pctl->active, active_release, + PERCPU_REF_ALLOW_REINIT, GFP_NOIO)) { + __free_page(page); + kfree(pctl); + return ERR_PTR(-ENOMEM); + } + + pctl->page = page; + pctl->state = page_address(page); + init_waitqueue_head(&pctl->wait); + return pctl; +} + +static unsigned int llbitmap_reserved_pages(struct llbitmap *llbitmap) +{ + return DIV_ROUND_UP(llbitmap->mddev->bitmap_info.space << SECTOR_SHIFT, + PAGE_SIZE); +} + +static int llbitmap_expand_pages(struct llbitmap *llbitmap, + unsigned long chunks) +{ + struct llbitmap_page_ctl **pctl; + unsigned int old_nr_pages = llbitmap->nr_pages; + unsigned int nr_pages = llbitmap_used_pages(llbitmap, chunks); + unsigned int i; + int ret; + + if (nr_pages <= old_nr_pages) + return 0; + + pctl = kcalloc(nr_pages, sizeof(*pctl), GFP_NOIO); + if (!pctl) + return -ENOMEM; + + if (llbitmap->pctl) + memcpy(pctl, llbitmap->pctl, + array_size(old_nr_pages, sizeof(*pctl))); + + for (i = old_nr_pages; i < nr_pages; i++) { + pctl[i] = llbitmap_alloc_page_ctl(llbitmap, i); + if (IS_ERR(pctl[i])) + goto err_alloc_ptr; + } + + kfree(llbitmap->pctl); + llbitmap->pctl = pctl; + llbitmap->nr_pages = nr_pages; + return 0; + +err_alloc_ptr: + ret = PTR_ERR(pctl[i]); + while (i-- > old_nr_pages) { + __free_page(pctl[i]->page); + percpu_ref_exit(&pctl[i]->active); + kfree(pctl[i]); + } + kfree(pctl); + return ret; +} + +static int llbitmap_alloc_pages(struct llbitmap *llbitmap) +{ + unsigned int used_pages = llbitmap_used_pages(llbitmap, llbitmap->chunks); + unsigned int nr_pages = max(used_pages, llbitmap_reserved_pages(llbitmap)); int i; - llbitmap->pctl = kmalloc_array(nr_pages, sizeof(void *), - GFP_KERNEL | __GFP_ZERO); + llbitmap->pctl = kcalloc(nr_pages, sizeof(*llbitmap->pctl), GFP_NOIO); if (!llbitmap->pctl) return -ENOMEM; - size = round_up(size, cache_line_size()); - pctl = kmalloc_array(nr_pages, size, GFP_KERNEL | __GFP_ZERO); - if (!pctl) { - kfree(llbitmap->pctl); - return -ENOMEM; - } - llbitmap->nr_pages = nr_pages; - for (i = 0; i < nr_pages; i++, pctl = (void *)pctl + size) { - struct page *page = llbitmap_read_page(llbitmap, i); + for (i = 0; i < nr_pages; i++) { + llbitmap->pctl[i] = llbitmap_alloc_page_ctl(llbitmap, i); + if (IS_ERR(llbitmap->pctl[i])) { + int ret = PTR_ERR(llbitmap->pctl[i]); - llbitmap->pctl[i] = pctl; - - if (IS_ERR(page)) { + llbitmap->pctl[i] = NULL; llbitmap_free_pages(llbitmap); - return PTR_ERR(page); + return ret; } - - if (percpu_ref_init(&pctl->active, active_release, - PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) { - __free_page(page); - llbitmap_free_pages(llbitmap); - return -ENOMEM; - } - - pctl->page = page; - pctl->state = page_address(page); - init_waitqueue_head(&pctl->wait); } return 0; @@ -703,6 +915,61 @@ static bool llbitmap_zero_all_disks(struct llbitmap *llbitmap) return true; } +static void llbitmap_mark_range(struct llbitmap *llbitmap, + unsigned long start, + unsigned long end, + enum llbitmap_state state) +{ + while (start <= end) { + llbitmap_write(llbitmap, state, start); + start++; + } +} + +static int llbitmap_prepare_resize(struct llbitmap *llbitmap, + unsigned long old_chunks, + unsigned long new_chunks, + unsigned long cache_chunks) +{ + int ret; + + llbitmap_flush(llbitmap->mddev); + ret = llbitmap_expand_pages(llbitmap, cache_chunks); + if (ret) + return ret; + if (new_chunks > old_chunks) + llbitmap_mark_range(llbitmap, old_chunks, new_chunks - 1, + BitUnwritten); + return 0; +} + +static enum llbitmap_state +llbitmap_rmerge_state(struct llbitmap *llbitmap, + enum llbitmap_state dst, + enum llbitmap_state src) +{ + bool level_456 = raid_is_456(llbitmap->mddev); + + if (dst == BitNeedSync || dst == BitSyncing || + src == BitNeedSync || src == BitSyncing) + return BitNeedSync; + + if (dst == BitDirty || src == BitDirty) + return BitDirty; + + /* + * Reshape generates valid target parity/data for both already-written + * and not-yet-written regions in the checkpointed range, so a mix of + * clean and unwritten still results in a clean destination bit. + */ + if (level_456 && ((dst == BitClean && src == BitUnwritten) || + (src == BitClean && dst == BitUnwritten))) + return BitClean; + if (dst == BitClean || src == BitClean) + return BitClean; + return BitUnwritten; +} + static void llbitmap_init_state(struct llbitmap *llbitmap) { struct mddev *mddev = llbitmap->mddev; @@ -745,7 +1012,10 @@ static enum llbitmap_state llbitmap_state_machine(struct llbitmap *llbitmap, llbitmap_init_state(llbitmap); return BitNone; } - + if (start >= llbitmap->chunks) + return BitNone; + if (end >= llbitmap->chunks) + end = llbitmap->chunks - 1; while (start <= end) { enum llbitmap_state c = llbitmap_read(llbitmap, start); @@ -789,6 +1059,7 @@ static enum llbitmap_state llbitmap_state_machine(struct llbitmap *llbitmap, if (state == BitNeedSync || state == BitNeedSyncUnwritten) need_resync = !mddev->degraded; else if (state == BitDirty && + !test_bit(BITMAP_SHUTDOWN, &llbitmap->flags) && !timer_pending(&llbitmap->pending_timer)) mod_timer(&llbitmap->pending_timer, jiffies + mddev->bitmap_info.daemon_sleep * HZ); @@ -918,9 +1189,11 @@ static int llbitmap_init(struct llbitmap *llbitmap) llbitmap->chunkshift = ffz(~chunksize); llbitmap->chunksize = chunksize; llbitmap->chunks = chunks; + llbitmap->sync_size = blocks; + llbitmap_refresh_reshape(llbitmap); mddev->bitmap_info.daemon_sleep = DEFAULT_DAEMON_SLEEP; - ret = llbitmap_cache_pages(llbitmap); + ret = llbitmap_alloc_pages(llbitmap); if (ret) return ret; @@ -938,6 +1211,7 @@ static int llbitmap_read_sb(struct llbitmap *llbitmap) unsigned long daemon_sleep; unsigned long chunksize; unsigned long events; + sector_t sync_size; struct page *sb_page; bitmap_super_t *sb; int ret = -EINVAL; @@ -981,12 +1255,20 @@ static int llbitmap_read_sb(struct llbitmap *llbitmap) else mddev->bitmap_info.space = mddev->bitmap_info.default_space; } - llbitmap->flags = le32_to_cpu(sb->state); + llbitmap->flags = le32_to_cpu(sb->state) & ~BIT(BITMAP_SHUTDOWN); if (test_and_clear_bit(BITMAP_FIRST_USE, &llbitmap->flags)) { ret = llbitmap_init(llbitmap); goto out_put_page; } + sync_size = le64_to_cpu(sb->sync_size); + if (!sync_size) + sync_size = mddev->resync_max_sectors; + if (sync_size > mddev->resync_max_sectors) { + pr_err("md/llbitmap: %s: sync_size %llu exceeds array sync size %llu", + mdname(mddev), sync_size, mddev->resync_max_sectors); + goto out_put_page; + } chunksize = le32_to_cpu(sb->chunksize); if (!is_power_of_2(chunksize)) { pr_err("md/llbitmap: %s: chunksize not a power of 2", @@ -994,10 +1276,10 @@ static int llbitmap_read_sb(struct llbitmap *llbitmap) goto out_put_page; } - if (chunksize < DIV_ROUND_UP_SECTOR_T(mddev->resync_max_sectors, + if (chunksize < DIV_ROUND_UP_SECTOR_T(sync_size, mddev->bitmap_info.space << SECTOR_SHIFT)) { pr_err("md/llbitmap: %s: chunksize too small %lu < %llu / %lu", - mdname(mddev), chunksize, mddev->resync_max_sectors, + mdname(mddev), chunksize, sync_size, mddev->bitmap_info.space); goto out_put_page; } @@ -1022,9 +1304,11 @@ static int llbitmap_read_sb(struct llbitmap *llbitmap) llbitmap->barrier_idle = DEFAULT_BARRIER_IDLE; llbitmap->chunksize = chunksize; - llbitmap->chunks = DIV_ROUND_UP_SECTOR_T(mddev->resync_max_sectors, chunksize); + llbitmap->chunks = DIV_ROUND_UP_SECTOR_T(sync_size, chunksize); llbitmap->chunkshift = ffz(~chunksize); - ret = llbitmap_cache_pages(llbitmap); + llbitmap->sync_size = sync_size; + llbitmap_refresh_reshape(llbitmap); + ret = llbitmap_alloc_pages(llbitmap); out_put_page: __free_page(sb_page); @@ -1037,6 +1321,9 @@ static void llbitmap_pending_timer_fn(struct timer_list *pending_timer) struct llbitmap *llbitmap = container_of(pending_timer, struct llbitmap, pending_timer); + if (test_bit(BITMAP_SHUTDOWN, &llbitmap->flags)) + return; + if (work_busy(&llbitmap->daemon_work)) { pr_warn("md/llbitmap: %s daemon_work not finished in %lu seconds\n", mdname(llbitmap->mddev), @@ -1057,8 +1344,12 @@ static void md_llbitmap_daemon_fn(struct work_struct *work) bool restart; int idx; + if (test_bit(BITMAP_SHUTDOWN, &llbitmap->flags)) + return; + if (llbitmap->mddev->degraded) return; + retry: start = 0; end = min(llbitmap->chunks, PAGE_SIZE - BITMAP_DATA_OFFSET) - 1; @@ -1066,14 +1357,14 @@ static void md_llbitmap_daemon_fn(struct work_struct *work) for (idx = 0; idx < llbitmap->nr_pages; idx++) { struct llbitmap_page_ctl *pctl = llbitmap->pctl[idx]; + bool flush = test_and_clear_bit(LLPageFlush, &pctl->flags); if (idx > 0) { start = end + 1; end = min(end + PAGE_SIZE, llbitmap->chunks - 1); } - if (!test_bit(LLPageFlush, &pctl->flags) && - time_before(jiffies, pctl->expire)) { + if (!flush && time_before(jiffies, pctl->expire)) { restart = true; continue; } @@ -1096,7 +1387,7 @@ static void md_llbitmap_daemon_fn(struct work_struct *work) goto retry; /* If some page is dirty but not expired, setup timer again */ - if (restart) + if (restart && !test_bit(BITMAP_SHUTDOWN, &llbitmap->flags)) mod_timer(&llbitmap->pending_timer, jiffies + llbitmap->mddev->bitmap_info.daemon_sleep * HZ); } @@ -1110,7 +1401,7 @@ static int llbitmap_create(struct mddev *mddev) if (ret) return ret; - llbitmap = kzalloc_obj(*llbitmap); + llbitmap = kzalloc_obj(*llbitmap, GFP_NOIO); if (!llbitmap) return -ENOMEM; @@ -1120,16 +1411,18 @@ static int llbitmap_create(struct mddev *mddev) timer_setup(&llbitmap->pending_timer, llbitmap_pending_timer_fn, 0); INIT_WORK(&llbitmap->daemon_work, md_llbitmap_daemon_fn); + rwlock_init(&llbitmap->reshape_lock); atomic_set(&llbitmap->behind_writes, 0); init_waitqueue_head(&llbitmap->behind_wait); mutex_lock(&mddev->bitmap_info.mutex); mddev->bitmap = llbitmap; ret = llbitmap_read_sb(llbitmap); + if (ret) + mddev->bitmap = NULL; mutex_unlock(&mddev->bitmap_info.mutex); if (ret) { kfree(llbitmap); - mddev->bitmap = NULL; } return ret; @@ -1138,34 +1431,86 @@ static int llbitmap_create(struct mddev *mddev) static int llbitmap_resize(struct mddev *mddev, sector_t blocks, int chunksize) { struct llbitmap *llbitmap = mddev->bitmap; + sector_t old_blocks = llbitmap->sync_size; + unsigned long old_chunks = llbitmap->chunks; unsigned long chunks; + unsigned long cache_chunks; + int ret = 0; + unsigned long bitmap_chunksize; + bool reshape; + bool quiesced = false; if (chunksize == 0) chunksize = llbitmap->chunksize; - /* If there is enough space, leave the chunksize unchanged. */ - chunks = DIV_ROUND_UP_SECTOR_T(blocks, chunksize); - while (chunks > mddev->bitmap_info.space << SECTOR_SHIFT) { - chunksize = chunksize << 1; - chunks = DIV_ROUND_UP_SECTOR_T(blocks, chunksize); + bitmap_chunksize = chunksize; + llbitmap_calculate_chunks(mddev, blocks, &bitmap_chunksize, &chunks); + + reshape = mddev->delta_disks || mddev->new_level != mddev->level || + mddev->new_layout != mddev->layout || + mddev->new_chunk_sectors != mddev->chunk_sectors; + if (!reshape && bitmap_chunksize != llbitmap->chunksize) + return -EOPNOTSUPP; + if (blocks == old_blocks && chunks == llbitmap->chunks) + return 0; + + if (mddev->pers->quiesce) { + mddev->pers->quiesce(mddev, 1); + quiesced = true; } - llbitmap->chunkshift = ffz(~chunksize); - llbitmap->chunksize = chunksize; - llbitmap->chunks = chunks; + mutex_lock(&mddev->bitmap_info.mutex); + cache_chunks = reshape ? max(old_chunks, chunks) : chunks; + ret = llbitmap_prepare_resize(llbitmap, old_chunks, chunks, cache_chunks); + if (ret) + goto out; + if (reshape) { + llbitmap->reshape_sync_size = blocks; + llbitmap->reshape_chunksize = bitmap_chunksize; + llbitmap->reshape_chunks = chunks; + llbitmap->chunks = max(old_chunks, chunks); + } else { + if (blocks < old_blocks && chunks < old_chunks) + llbitmap_mark_range(llbitmap, chunks, old_chunks - 1, + BitUnwritten); + mddev->bitmap_info.chunksize = bitmap_chunksize; + llbitmap->chunks = chunks; + llbitmap->sync_size = blocks; + llbitmap_refresh_reshape(llbitmap); + llbitmap_update_sb(llbitmap); + } + __llbitmap_flush(mddev); + mutex_unlock(&mddev->bitmap_info.mutex); + if (quiesced) + mddev->pers->quiesce(mddev, 0); return 0; + +out: + mutex_unlock(&mddev->bitmap_info.mutex); + if (quiesced) + mddev->pers->quiesce(mddev, 0); + return ret; } static int llbitmap_load(struct mddev *mddev) { enum llbitmap_action action = BitmapActionReload; struct llbitmap *llbitmap = mddev->bitmap; + int ret; if (test_and_clear_bit(BITMAP_STALE, &llbitmap->flags)) action = BitmapActionStale; + mutex_lock(&mddev->bitmap_info.mutex); + llbitmap_refresh_reshape(llbitmap); + ret = llbitmap_expand_pages(llbitmap, llbitmap->chunks); + if (ret) { + mutex_unlock(&mddev->bitmap_info.mutex); + return ret; + } llbitmap_state_machine(llbitmap, 0, llbitmap->chunks - 1, action); + mutex_unlock(&mddev->bitmap_info.mutex); return 0; } @@ -1178,7 +1523,9 @@ static void llbitmap_destroy(struct mddev *mddev) mutex_lock(&mddev->bitmap_info.mutex); - timer_delete_sync(&llbitmap->pending_timer); + set_bit(BITMAP_SHUTDOWN, &llbitmap->flags); + timer_shutdown_sync(&llbitmap->pending_timer); + cancel_work_sync(&llbitmap->daemon_work); flush_workqueue(md_llbitmap_io_wq); flush_workqueue(md_llbitmap_unplug_wq); @@ -1188,6 +1535,37 @@ static void llbitmap_destroy(struct mddev *mddev) mutex_unlock(&mddev->bitmap_info.mutex); } +static bool llbitmap_map_previous(struct llbitmap *llbitmap, sector_t offset, + unsigned long sectors) +{ + struct mddev *mddev = llbitmap->mddev; + sector_t boundary = READ_ONCE(mddev->reshape_position); + + if (boundary == MaxSector) + return false; + + WARN_ON_ONCE(sectors && offset < boundary && offset + sectors > boundary); + + return mddev->reshape_backwards ? offset < boundary : offset >= boundary; +} + +static void llbitmap_prepare_range(struct mddev *mddev, sector_t *offset, + unsigned long *sectors, bool discard) +{ + struct llbitmap *llbitmap = mddev->bitmap; + bool previous; + + if (!llbitmap) + return; + + previous = llbitmap_map_previous(llbitmap, *offset, *sectors); + llbitmap_map_layout(llbitmap, offset, sectors, previous); + if (discard) + llbitmap_encode_discard_range(llbitmap, offset, sectors, previous); + else + llbitmap_encode_range(llbitmap, offset, sectors, previous); +} + static void llbitmap_start_write(struct mddev *mddev, sector_t offset, unsigned long sectors) { @@ -1202,7 +1580,9 @@ static void llbitmap_start_write(struct mddev *mddev, sector_t offset, page_start++; } + read_lock(&llbitmap->reshape_lock); llbitmap_state_machine(llbitmap, start, end, BitmapActionStartwrite); + read_unlock(&llbitmap->reshape_lock); } static void llbitmap_end_write(struct mddev *mddev, sector_t offset, @@ -1234,7 +1614,9 @@ static void llbitmap_start_discard(struct mddev *mddev, sector_t offset, page_start++; } + read_lock(&llbitmap->reshape_lock); llbitmap_state_machine(llbitmap, start, end, BitmapActionDiscard); + read_unlock(&llbitmap->reshape_lock); } static void llbitmap_end_discard(struct mddev *mddev, sector_t offset, @@ -1354,7 +1736,11 @@ static bool llbitmap_blocks_synced(struct mddev *mddev, sector_t offset) { struct llbitmap *llbitmap = mddev->bitmap; unsigned long p = offset >> llbitmap->chunkshift; - enum llbitmap_state c = llbitmap_read(llbitmap, p); + enum llbitmap_state c; + + if (p >= llbitmap->chunks) + return false; + c = llbitmap_read(llbitmap, p); return c == BitClean || c == BitDirty || c == BitCleanUnwritten; } @@ -1364,7 +1750,19 @@ static sector_t llbitmap_skip_sync_blocks(struct mddev *mddev, sector_t offset) struct llbitmap *llbitmap = mddev->bitmap; unsigned long p = offset >> llbitmap->chunkshift; int blocks = llbitmap->chunksize - (offset & (llbitmap->chunksize - 1)); - enum llbitmap_state c = llbitmap_read(llbitmap, p); + enum llbitmap_state c; + + if (p >= llbitmap->chunks) + return 0; + c = llbitmap_read(llbitmap, p); + + /* + * Reshape progress is tracked by array metadata rather than llbitmap. + * Skipping reshape ranges from stale bitmap state can lose data after a + * restart before the corresponding bits are checkpointed to disk. + */ + if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) + return 0; /* always skip unwritten blocks */ if (c == BitUnwritten) @@ -1409,6 +1807,8 @@ static bool llbitmap_start_sync(struct mddev *mddev, sector_t offset, * if md_do_sync() loop more times. */ *blocks = llbitmap->chunksize - (offset & (llbitmap->chunksize - 1)); + if (p >= llbitmap->chunks) + return false; state = llbitmap_state_machine(llbitmap, p, p, BitmapActionStartsync); return state == BitSyncing || state == BitSyncingUnwritten; } @@ -1450,22 +1850,27 @@ static void llbitmap_cond_end_sync(struct mddev *mddev, sector_t sector, bool force) { struct llbitmap *llbitmap = mddev->bitmap; + sector_t complete; if (sector == 0) { llbitmap->last_end_sync = jiffies; return; } - if (time_before(jiffies, llbitmap->last_end_sync + - HZ * mddev->bitmap_info.daemon_sleep)) + if (!force && time_before(jiffies, llbitmap->last_end_sync + + HZ * mddev->bitmap_info.daemon_sleep)) return; wait_event(mddev->recovery_wait, !atomic_read(&mddev->recovery_active)); mddev->curr_resync_completed = sector; set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags); - llbitmap_state_machine(llbitmap, 0, sector >> llbitmap->chunkshift, - BitmapActionEndsync); + + complete = round_down(sector, llbitmap->chunksize); + if (complete) + llbitmap_state_machine(llbitmap, 0, + (complete >> llbitmap->chunkshift) - 1, + BitmapActionEndsync); __llbitmap_flush(mddev); llbitmap->last_end_sync = jiffies; @@ -1485,6 +1890,210 @@ static void llbitmap_dirty_bits(struct mddev *mddev, unsigned long s, llbitmap_state_machine(mddev->bitmap, s, e, BitmapActionStartwrite); } +static int llbitmap_reshape_can_start(struct mddev *mddev) +{ + struct llbitmap *llbitmap = mddev->bitmap; + unsigned long chunk; + int ret = 0; + + if (!llbitmap) + return 0; + + mutex_lock(&mddev->bitmap_info.mutex); + for (chunk = 0; chunk < llbitmap->chunks; chunk++) { + enum llbitmap_state state = llbitmap_read(llbitmap, chunk); + + if (state == BitNeedSync || state == BitSyncing) { + ret = -EBUSY; + break; + } + } + mutex_unlock(&mddev->bitmap_info.mutex); + + return ret; +} + +struct llbitmap_reshape_range { + sector_t offset; + unsigned long sectors; + sector_t start; + sector_t end; +}; + +static enum llbitmap_state +llbitmap_reshape_init_dst(struct llbitmap *llbitmap, unsigned long dst, + const struct llbitmap_reshape_range *new) +{ + u64 bit_start = (u64)dst * llbitmap->reshape_chunksize; + u64 bit_end = bit_start + llbitmap->reshape_chunksize; + + if (!llbitmap->mddev->reshape_backwards) + return bit_start < new->offset ? llbitmap_read(llbitmap, dst) : + BitUnwritten; + return bit_end > new->end ? llbitmap_read(llbitmap, dst) : BitUnwritten; +} + +static void llbitmap_reshape_dst_range(struct llbitmap *llbitmap, + unsigned long dst, + const struct llbitmap_reshape_range *new, + struct llbitmap_reshape_range *dst_range) +{ + sector_t dst_bit_start = (sector_t)dst * llbitmap->reshape_chunksize; + + dst_range->start = max(dst_bit_start, new->offset); + dst_range->end = min(dst_bit_start + llbitmap->reshape_chunksize, + new->end); + dst_range->offset = dst_range->start; + dst_range->sectors = dst_range->end - dst_range->start; +} + +static void llbitmap_reshape_map_range(struct llbitmap *llbitmap, + sector_t lo, sector_t hi, + bool previous, + struct llbitmap_reshape_range *range) +{ + range->offset = lo; + range->sectors = hi - lo; + llbitmap_map_layout(llbitmap, &range->offset, &range->sectors, previous); + range->start = range->offset; + range->end = range->offset + range->sectors; +} + +static bool llbitmap_reshape_src_range(const struct llbitmap_reshape_range *old, + const struct llbitmap_reshape_range *new, + const struct llbitmap_reshape_range *dst, + struct llbitmap_reshape_range *src) +{ + if (!old->sectors) + return false; + + src->start = old->offset + + mul_u64_u64_div_u64(dst->start - new->offset, + old->sectors, new->sectors); + src->end = old->offset + + mul_u64_u64_div_u64_roundup(dst->end - new->offset, + old->sectors, new->sectors); + if (src->end > old->end) + src->end = old->end; + src->offset = src->start; + src->sectors = src->end - src->start; + + return src->sectors; +} + +static enum llbitmap_state llbitmap_rmerge_src(struct llbitmap *llbitmap, + enum llbitmap_state state, + const struct llbitmap_reshape_range *src) +{ + unsigned long bit = div64_u64(src->start, llbitmap->chunksize); + unsigned long end = div64_u64(src->end - 1, llbitmap->chunksize); + + while (bit <= end) { + enum llbitmap_state src_state = llbitmap_read(llbitmap, bit); + + state = llbitmap_rmerge_state(llbitmap, state, src_state); + bit++; + } + + return state; +} + +static void llbitmap_reshape_merge(struct llbitmap *llbitmap, + const struct llbitmap_reshape_range *old, + const struct llbitmap_reshape_range *new) +{ + unsigned long dst_start; + unsigned long dst_end; + unsigned long dst; + bool backwards = false; + + if (!new->sectors) + return; + + dst_start = div64_u64(new->offset, llbitmap->reshape_chunksize); + dst_end = div64_u64(new->end - 1, llbitmap->reshape_chunksize); + if (old->sectors) { + unsigned long src_start = div64_u64(old->offset, + llbitmap->chunksize); + unsigned long src_end = div64_u64(old->end - 1, + llbitmap->chunksize); + + backwards = src_start < dst_start && src_end >= dst_start; + } + + dst = backwards ? dst_end : dst_start; + while (true) { + struct llbitmap_reshape_range dst_range; + struct llbitmap_reshape_range src; + enum llbitmap_state state; + + llbitmap_reshape_dst_range(llbitmap, dst, new, &dst_range); + state = llbitmap_reshape_init_dst(llbitmap, dst, new); + if (llbitmap_reshape_src_range(old, new, &dst_range, &src)) + state = llbitmap_rmerge_src(llbitmap, state, &src); + else + state = llbitmap_rmerge_state(llbitmap, state, BitUnwritten); + llbitmap_write(llbitmap, state, dst); + if (dst == (backwards ? dst_start : dst_end)) + break; + if (backwards) + dst--; + else + dst++; + } +} + +static void llbitmap_reshape_finish(struct mddev *mddev) +{ + struct llbitmap *llbitmap = mddev->bitmap; + + if (mddev->pers->quiesce) + mddev->pers->quiesce(mddev, 1); + + mutex_lock(&mddev->bitmap_info.mutex); + llbitmap_flush(mddev); + + llbitmap->chunksize = llbitmap->reshape_chunksize; + llbitmap->chunkshift = ffz(~llbitmap->chunksize); + llbitmap->chunks = llbitmap->reshape_chunks; + llbitmap->sync_size = llbitmap->reshape_sync_size; + llbitmap_refresh_reshape(llbitmap); + mddev->bitmap_info.chunksize = llbitmap->chunksize; + llbitmap_update_sb(llbitmap); + __llbitmap_flush(mddev); + mutex_unlock(&mddev->bitmap_info.mutex); + + if (mddev->pers->quiesce) + mddev->pers->quiesce(mddev, 0); +} + +static void llbitmap_reshape_mark(struct mddev *mddev, sector_t old_pos, + sector_t new_pos) +{ + struct llbitmap *llbitmap = mddev->bitmap; + sector_t lo; + sector_t hi; + struct llbitmap_reshape_range old; + struct llbitmap_reshape_range new; + + if (!llbitmap || old_pos == new_pos) + return; + + lo = min(old_pos, new_pos); + hi = max(old_pos, new_pos); + if (!hi) + return; + + llbitmap_reshape_map_range(llbitmap, lo, hi, true, &old); + llbitmap_reshape_map_range(llbitmap, lo, hi, false, &new); + if (!new.sectors) + return; + + write_lock(&llbitmap->reshape_lock); + llbitmap_reshape_merge(llbitmap, &old, &new); + write_unlock(&llbitmap->reshape_lock); +} + static void llbitmap_write_sb(struct llbitmap *llbitmap) { int nr_blocks = DIV_ROUND_UP(BITMAP_DATA_OFFSET, llbitmap->io_size); @@ -1517,9 +2126,9 @@ static void llbitmap_update_sb(void *data) sb = kmap_local_page(sb_page); sb->events = cpu_to_le64(mddev->events); - sb->state = cpu_to_le32(llbitmap->flags); + sb->state = cpu_to_le32(llbitmap->flags & ~BIT(BITMAP_SHUTDOWN)); sb->chunksize = cpu_to_le32(llbitmap->chunksize); - sb->sync_size = cpu_to_le64(mddev->resync_max_sectors); + sb->sync_size = cpu_to_le64(llbitmap->sync_size); sb->events_cleared = cpu_to_le64(llbitmap->events_cleared); sb->sectors_reserved = cpu_to_le32(mddev->bitmap_info.space); sb->daemon_sleep = cpu_to_le32(mddev->bitmap_info.daemon_sleep); @@ -1537,6 +2146,7 @@ static int llbitmap_get_stats(void *data, struct md_bitmap_stats *stats) stats->missing_pages = 0; stats->pages = llbitmap->nr_pages; stats->file_pages = llbitmap->nr_pages; + stats->sync_size = llbitmap->sync_size; stats->behind_writes = atomic_read(&llbitmap->behind_writes); stats->behind_wait = wq_has_sleeper(&llbitmap->behind_wait); @@ -1574,19 +2184,13 @@ static void llbitmap_end_behind_write(struct mddev *mddev) wake_up(&llbitmap->behind_wait); } -static bool llbitmap_wait_behind_writes(struct mddev *mddev, bool nowait) +static void llbitmap_wait_behind_writes(struct mddev *mddev) { struct llbitmap *llbitmap = mddev->bitmap; - if (llbitmap && atomic_read(&llbitmap->behind_writes) > 0) { - if (nowait) - return false; - + if (llbitmap && atomic_read(&llbitmap->behind_writes) > 0) wait_event(llbitmap->behind_wait, atomic_read(&llbitmap->behind_writes) == 0); - } - - return true; } static ssize_t bits_show(struct mddev *mddev, char *page) @@ -1780,6 +2384,10 @@ static struct bitmap_operations llbitmap_ops = { .update_sb = llbitmap_update_sb, .get_stats = llbitmap_get_stats, .dirty_bits = llbitmap_dirty_bits, + .prepare_range = llbitmap_prepare_range, + .reshape_finish = llbitmap_reshape_finish, + .reshape_can_start = llbitmap_reshape_can_start, + .reshape_mark = llbitmap_reshape_mark, .write_all = llbitmap_write_all, .groups = md_llbitmap_groups, diff --git a/drivers/md/md.c b/drivers/md/md.c index d1465bcd86c8..680b34a63cb3 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -233,23 +233,22 @@ static int rdev_need_serial(struct md_rdev *rdev) void mddev_create_serial_pool(struct mddev *mddev, struct md_rdev *rdev) { int ret = 0; + unsigned int noio_flags; - if (rdev && !rdev_need_serial(rdev) && + if (!test_bit(MD_SERIALIZE_POLICY, &mddev->flags) && + rdev && !rdev_need_serial(rdev) && !test_bit(CollisionCheck, &rdev->flags)) return; + noio_flags = memalloc_noio_save(); if (!rdev) ret = rdevs_init_serial(mddev); else ret = rdev_init_serial(rdev); if (ret) - return; + goto out; if (mddev->serial_info_pool == NULL) { - /* - * already in memalloc noio context by - * mddev_suspend() - */ mddev->serial_info_pool = mempool_create_kmalloc_pool(NR_SERIAL_INFOS, sizeof(struct serial_info)); @@ -258,6 +257,8 @@ void mddev_create_serial_pool(struct mddev *mddev, struct md_rdev *rdev) pr_err("can't alloc memory pool for serialization\n"); } } +out: + memalloc_noio_restore(noio_flags); } /* @@ -516,9 +517,6 @@ int mddev_suspend(struct mddev *mddev, bool interruptible) */ WRITE_ONCE(mddev->suspended, mddev->suspended + 1); - /* restrict memory reclaim I/O during raid array is suspend */ - mddev->noio_flag = memalloc_noio_save(); - mutex_unlock(&mddev->suspend_mutex); return 0; } @@ -535,9 +533,6 @@ static void __mddev_resume(struct mddev *mddev, bool recovery_needed) return; } - /* entred the memalloc scope from mddev_suspend() */ - memalloc_noio_restore(mddev->noio_flag); - percpu_ref_resurrect(&mddev->active_io); wake_up(&mddev->sb_wait); @@ -1920,6 +1915,13 @@ static int super_1_load(struct md_rdev *rdev, struct md_rdev *refdev, int minor_ rdev->bb_page, REQ_OP_READ, true)) return -EIO; bbp = (__le64 *)page_address(rdev->bb_page); + + /* check for badblocks api. */ + if (sb->bblog_shift >= BITS_PER_TYPE(sector_t)) { + pr_err("md: %pg: bogus bblog_shift %u for badblocks.\n", + rdev->bdev, sb->bblog_shift); + return -EINVAL; + } rdev->badblocks.shift = sb->bblog_shift; for (i = 0 ; i < (sectors << (9-3)) ; i++, bbp++) { u64 bb = le64_to_cpu(*bbp); @@ -4047,6 +4049,7 @@ level_store(struct mddev *mddev, const char *buf, size_t len) char clevel[16]; ssize_t rv; size_t slen = len; + unsigned int noio_flags; struct md_personality *pers, *oldpers; long level; void *priv, *oldpriv; @@ -4058,6 +4061,7 @@ level_store(struct mddev *mddev, const char *buf, size_t len) rv = mddev_suspend_and_lock(mddev); if (rv) return rv; + noio_flags = memalloc_noio_save(); if (mddev->pers == NULL) { memcpy(mddev->clevel, buf, slen); @@ -4233,6 +4237,7 @@ level_store(struct mddev *mddev, const char *buf, size_t len) md_new_event(); rv = len; out_unlock: + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); return rv; } @@ -4412,6 +4417,7 @@ static ssize_t raid_disks_store(struct mddev *mddev, const char *buf, size_t len) { unsigned int n; + unsigned int noio_flags; int err; err = kstrtouint(buf, 10, &n); @@ -4421,6 +4427,7 @@ raid_disks_store(struct mddev *mddev, const char *buf, size_t len) err = mddev_suspend_and_lock(mddev); if (err) return err; + noio_flags = memalloc_noio_save(); if (mddev->pers) { if (n != mddev->raid_disks) err = update_raid_disks(mddev, n); @@ -4444,6 +4451,7 @@ raid_disks_store(struct mddev *mddev, const char *buf, size_t len) } else mddev->raid_disks = n; out_unlock: + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); return err ? err : len; } @@ -4824,6 +4832,7 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len) int minor; dev_t dev; struct md_rdev *rdev; + unsigned int noio_flags; int err; if (!*buf || *e != ':' || !e[1] || e[1] == '\n') @@ -4839,6 +4848,7 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len) err = mddev_suspend_and_lock(mddev); if (err) return err; + noio_flags = memalloc_noio_save(); if (mddev->persistent) { rdev = md_import_device(dev, mddev->major_version, mddev->minor_version); @@ -4857,6 +4867,7 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len) rdev = md_import_device(dev, -1, -1); if (IS_ERR(rdev)) { + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); return PTR_ERR(rdev); } @@ -4864,6 +4875,7 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len) out: if (err) export_rdev(rdev); + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); if (!err) md_new_event(); @@ -5263,21 +5275,28 @@ action_store(struct mddev *mddev, const char *page, size_t len) if (!mddev->pers || !mddev->pers->sync_request) return -EINVAL; + action = md_sync_action_by_name(page); + if (action == ACTION_RESHAPE) { + ret = mddev_suspend(mddev, true); + if (ret) + return ret; + } retry: if (work_busy(&mddev->sync_work)) flush_work(&mddev->sync_work); ret = mddev_lock(mddev); - if (ret) + if (ret) { + if (action == ACTION_RESHAPE) + mddev_resume(mddev); return ret; + } if (work_busy(&mddev->sync_work)) { mddev_unlock(mddev); goto retry; } - action = md_sync_action_by_name(page); - /* TODO: mdadm rely on "idle" to start sync_thread. */ if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) { switch (action) { @@ -5347,6 +5366,8 @@ action_store(struct mddev *mddev, const char *page, size_t len) out: mddev_unlock(mddev); + if (action == ACTION_RESHAPE) + mddev_resume(mddev); return ret; } @@ -6283,7 +6304,7 @@ void md_init_stacking_limits(struct queue_limits *lim) { blk_set_stacking_limits(lim); lim->features = BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA | - BLK_FEAT_IO_STAT | BLK_FEAT_NOWAIT; + BLK_FEAT_IO_STAT; } EXPORT_SYMBOL_GPL(md_init_stacking_limits); @@ -6631,7 +6652,6 @@ int md_run(struct mddev *mddev) int err; struct md_rdev *rdev; struct md_personality *pers; - bool nowait = true; if (list_empty(&mddev->disks)) /* cannot run an array with no devices.. */ @@ -6702,7 +6722,6 @@ int md_run(struct mddev *mddev) } } sysfs_notify_dirent_safe(rdev->sysfs_state); - nowait = nowait && bdev_nowait(rdev->bdev); } pers = get_pers(mddev->level, mddev->clevel); @@ -7050,7 +7069,7 @@ EXPORT_SYMBOL_GPL(md_stop_writes); static void mddev_detach(struct mddev *mddev) { if (md_bitmap_enabled(mddev, false)) - mddev->bitmap_ops->wait_behind_writes(mddev, false); + mddev->bitmap_ops->wait_behind_writes(mddev); if (mddev->pers && mddev->pers->quiesce && !is_md_suspended(mddev)) { mddev->pers->quiesce(mddev, 1); mddev->pers->quiesce(mddev, 0); @@ -7066,8 +7085,8 @@ static void __md_stop(struct mddev *mddev) { struct md_personality *pers = mddev->pers; - md_bitmap_destroy(mddev); mddev_detach(mddev); + md_bitmap_destroy(mddev); spin_lock(&mddev->lock); mddev->pers = NULL; spin_unlock(&mddev->lock); @@ -8324,8 +8343,10 @@ static int md_ioctl(struct block_device *bdev, blk_mode_t mode, unsigned int cmd, unsigned long arg) { int err = 0; + unsigned int noio_flags = 0; void __user *argp = (void __user *)arg; struct mddev *mddev = NULL; + bool suspend; err = md_ioctl_valid(cmd); if (err) @@ -8375,13 +8396,15 @@ static int md_ioctl(struct block_device *bdev, blk_mode_t mode, if (!md_is_rdwr(mddev)) flush_work(&mddev->sync_work); - err = md_ioctl_need_suspend(cmd) ? mddev_suspend_and_lock(mddev) : - mddev_lock(mddev); + suspend = md_ioctl_need_suspend(cmd); + err = suspend ? mddev_suspend_and_lock(mddev) : mddev_lock(mddev); if (err) { pr_debug("md: ioctl lock interrupted, reason %d, cmd %d\n", err, cmd); goto out; } + if (suspend) + noio_flags = memalloc_noio_save(); if (cmd == SET_ARRAY_INFO) { err = __md_set_array_info(mddev, argp); @@ -8506,8 +8529,12 @@ static int md_ioctl(struct block_device *bdev, blk_mode_t mode, err != -EINVAL) mddev->hold_active = 0; - md_ioctl_need_suspend(cmd) ? mddev_unlock_and_resume(mddev) : - mddev_unlock(mddev); + if (suspend) { + memalloc_noio_restore(noio_flags); + mddev_unlock_and_resume(mddev); + } else { + mddev_unlock(mddev); + } out: if (cmd == STOP_ARRAY_RO || (err && cmd == STOP_ARRAY)) @@ -9350,6 +9377,10 @@ void md_submit_discard_bio(struct mddev *mddev, struct md_rdev *rdev, { struct bio *discard_bio = NULL; + /* Discard is optional, so silently skip members that do not support it. */ + if (unlikely(!bdev_max_discard_sectors(rdev->bdev))) + return; + __blkdev_issue_discard(rdev->bdev, start, size, GFP_NOIO, &discard_bio); if (!discard_bio) return; @@ -9361,17 +9392,62 @@ void md_submit_discard_bio(struct mddev *mddev, struct md_rdev *rdev, } EXPORT_SYMBOL_GPL(md_submit_discard_bio); +struct bio *mddev_bio_split_at_reshape_offset(struct mddev *mddev, + struct bio *bio, + unsigned int *max_sectors, + struct bio_set *bs) +{ + sector_t boundary; + sector_t start; + sector_t end; + unsigned int split_sectors; + + split_sectors = bio_sectors(bio); + if (max_sectors && *max_sectors && *max_sectors < split_sectors) + split_sectors = *max_sectors; + + if (!test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) + goto split; + + boundary = READ_ONCE(mddev->reshape_position); + start = bio->bi_iter.bi_sector; + end = bio_end_sector(bio); + if (start >= boundary || end <= boundary) + goto split; + + if (boundary - start < split_sectors) + split_sectors = boundary - start; + +split: + if (max_sectors) + *max_sectors = split_sectors; + if (split_sectors < bio_sectors(bio)) { + bio = bio_submit_split_bioset(bio, split_sectors, bs); + if (bio) + bio->bi_opf |= REQ_NOMERGE; + } + + return bio; +} +EXPORT_SYMBOL_GPL(mddev_bio_split_at_reshape_offset); + +static void md_bitmap_prepare_range(struct mddev *mddev, sector_t *offset, + unsigned long *sectors, bool discard) +{ + mddev->bitmap_ops->prepare_range(mddev, offset, sectors, discard); +} + static void md_bitmap_start(struct mddev *mddev, struct md_io_clone *md_io_clone) { - md_bitmap_fn *fn = unlikely(md_io_clone->rw == STAT_DISCARD) ? - mddev->bitmap_ops->start_discard : + bool discard = md_io_clone->rw == STAT_DISCARD; + md_bitmap_fn *fn = discard ? mddev->bitmap_ops->start_discard : mddev->bitmap_ops->start_write; - if (mddev->pers->bitmap_sector) - mddev->pers->bitmap_sector(mddev, &md_io_clone->offset, - &md_io_clone->sectors); - + md_bitmap_prepare_range(mddev, &md_io_clone->offset, + &md_io_clone->sectors, discard); + if (!md_io_clone->sectors) + return; fn(mddev, md_io_clone->offset, md_io_clone->sectors); } @@ -9392,7 +9468,8 @@ static void md_end_clone_io(struct bio *bio) struct mddev *mddev = md_io_clone->mddev; struct completion *reshape_completion = bio->bi_private; - if (bio_data_dir(orig_bio) == WRITE && md_bitmap_enabled(mddev, false)) + if (bio_data_dir(orig_bio) == WRITE && md_io_clone->sectors && + md_bitmap_enabled(mddev, false)) md_bitmap_end(mddev, md_io_clone); if (bio->bi_status && !orig_bio->bi_status) @@ -9419,10 +9496,14 @@ static void md_clone_bio(struct mddev *mddev, struct bio **bio) md_io_clone = container_of(clone, struct md_io_clone, bio_clone); md_io_clone->orig_bio = *bio; md_io_clone->mddev = mddev; + md_io_clone->sectors = 0; if (blk_queue_io_stat(bdev->bd_disk->queue)) md_io_clone->start_time = bio_start_io_acct(*bio); + else + md_io_clone->start_time = 0; - if (bio_data_dir(*bio) == WRITE && md_bitmap_enabled(mddev, false)) { + if (bio_data_dir(*bio) == WRITE && bio_sectors(*bio) && + md_bitmap_enabled(mddev, false)) { md_io_clone->offset = (*bio)->bi_iter.bi_sector; md_io_clone->sectors = bio_sectors(*bio); md_io_clone->rw = op_stat_group(bio_op(*bio)); @@ -9881,8 +9962,10 @@ void md_do_sync(struct md_thread *thread) */ md_new_event(); - if (last_check + window > io_sectors || j == max_sectors) + if (last_check + window > io_sectors || j == max_sectors) { + cond_resched(); continue; + } last_check = io_sectors; repeat: @@ -10175,19 +10258,34 @@ static void md_start_sync(struct work_struct *ws) struct mddev *mddev = container_of(ws, struct mddev, sync_work); int spares = 0; bool suspend = false; + unsigned int noio_flags = 0; char *name; /* * If reshape is still in progress, spares won't be added or removed * from conf until reshape is done. */ - if (mddev->reshape_position == MaxSector && + if ((mddev->reshape_position == MaxSector || !md_is_rdwr(mddev)) && md_spares_need_change(mddev)) { suspend = true; mddev_suspend(mddev, false); + noio_flags = memalloc_noio_save(); } mddev_lock_nointr(mddev); + + /* + * The spare configuration can change before reconfig_mutex is acquired. + * Recheck while holding the lock and suspend if needed. + */ + if (!suspend && (mddev->reshape_position == MaxSector || !md_is_rdwr(mddev)) && + md_spares_need_change(mddev)) { + mddev_unlock(mddev); + mddev_suspend_and_lock_nointr(mddev); + suspend = true; + noio_flags = memalloc_noio_save(); + } + if (!md_is_rdwr(mddev)) { /* * On a read-only array we can: @@ -10231,8 +10329,10 @@ static void md_start_sync(struct work_struct *ws) * https://bugzilla.kernel.org/show_bug.cgi?id=218200 * Therefore, use __mddev_resume(mddev, false). */ - if (suspend) + if (suspend) { + memalloc_noio_restore(noio_flags); __mddev_resume(mddev, false); + } md_wakeup_thread(mddev->sync_thread); sysfs_notify_dirent_safe(mddev->sysfs_action); md_new_event(); @@ -10251,8 +10351,10 @@ static void md_start_sync(struct work_struct *ws) * https://bugzilla.kernel.org/show_bug.cgi?id=218200 * Therefore, use __mddev_resume(mddev, false). */ - if (suspend) + if (suspend) { + memalloc_noio_restore(noio_flags); __mddev_resume(mddev, false); + } wake_up(&resync_wait); if (test_and_clear_bit(MD_RECOVERY_RECOVER, &mddev->recovery) && @@ -10553,7 +10655,7 @@ EXPORT_SYMBOL(md_finish_reshape); /* Bad block management */ /* Returns true on success, false on failure */ -bool rdev_set_badblocks(struct md_rdev *rdev, sector_t s, int sectors, +bool rdev_set_badblocks(struct md_rdev *rdev, sector_t s, sector_t sectors, int is_new) { struct mddev *mddev = rdev->mddev; @@ -10593,7 +10695,7 @@ bool rdev_set_badblocks(struct md_rdev *rdev, sector_t s, int sectors, } EXPORT_SYMBOL_GPL(rdev_set_badblocks); -void rdev_clear_badblocks(struct md_rdev *rdev, sector_t s, int sectors, +void rdev_clear_badblocks(struct md_rdev *rdev, sector_t s, sector_t sectors, int is_new) { if (is_new) diff --git a/drivers/md/md.h b/drivers/md/md.h index d8daf0f75cbb..b6d2e8929a0f 100644 --- a/drivers/md/md.h +++ b/drivers/md/md.h @@ -311,7 +311,7 @@ static inline int is_badblock(struct md_rdev *rdev, sector_t s, sector_t sectors } static inline int rdev_has_badblock(struct md_rdev *rdev, sector_t s, - int sectors) + sector_t sectors) { sector_t first_bad; sector_t bad_sectors; @@ -319,9 +319,9 @@ static inline int rdev_has_badblock(struct md_rdev *rdev, sector_t s, return is_badblock(rdev, s, sectors, &first_bad, &bad_sectors); } -extern bool rdev_set_badblocks(struct md_rdev *rdev, sector_t s, int sectors, +extern bool rdev_set_badblocks(struct md_rdev *rdev, sector_t s, sector_t sectors, int is_new); -extern void rdev_clear_badblocks(struct md_rdev *rdev, sector_t s, int sectors, +extern void rdev_clear_badblocks(struct md_rdev *rdev, sector_t s, sector_t sectors, int is_new); struct md_cluster_info; struct md_cluster_operations; @@ -621,7 +621,6 @@ struct mddev { struct md_cluster_info *cluster_info; struct md_cluster_operations *cluster_ops; unsigned int good_device_nr; /* good device num within cluster raid */ - unsigned int noio_flag; /* for memalloc scope API */ /* * Temporarily store rdev that will be finally removed when @@ -798,6 +797,10 @@ struct md_personality /* convert io ranges from array to bitmap */ void (*bitmap_sector)(struct mddev *mddev, sector_t *offset, unsigned long *sectors); + void (*bitmap_sector_map)(struct mddev *mddev, sector_t *offset, + unsigned long *sectors, bool previous); + sector_t (*bitmap_sync_size)(struct mddev *mddev, bool previous); + sector_t (*bitmap_array_sectors)(struct mddev *mddev, bool previous); }; struct md_sysfs_entry { @@ -921,6 +924,10 @@ extern void md_error(struct mddev *mddev, struct md_rdev *rdev); extern void md_finish_reshape(struct mddev *mddev); void md_submit_discard_bio(struct mddev *mddev, struct md_rdev *rdev, struct bio *bio, sector_t start, sector_t size); +struct bio *mddev_bio_split_at_reshape_offset(struct mddev *mddev, + struct bio *bio, + unsigned int *max_sectors, + struct bio_set *bs); void md_account_bio(struct mddev *mddev, struct bio **bio); extern bool __must_check md_flush_request(struct mddev *mddev, struct bio *bio); diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c index 2c000b3a5f49..35e103f0c2c3 100644 --- a/drivers/md/raid0.c +++ b/drivers/md/raid0.c @@ -385,6 +385,7 @@ static int raid0_set_limits(struct mddev *mddev) int err; md_init_stacking_limits(&lim); + lim.features |= BLK_FEAT_NOWAIT; lim.max_hw_sectors = mddev->chunk_sectors; lim.max_write_zeroes_sectors = mddev->chunk_sectors; lim.max_hw_wzeroes_unmap_sectors = mddev->chunk_sectors; diff --git a/drivers/md/raid1-10.c b/drivers/md/raid1-10.c index 56a56a4da4f8..3b0e230692ba 100644 --- a/drivers/md/raid1-10.c +++ b/drivers/md/raid1-10.c @@ -290,9 +290,8 @@ static inline bool raid1_should_read_first(struct mddev *mddev, } /* - * bio with REQ_RAHEAD or REQ_NOWAIT can fail at anytime, before such IO is - * submitted to the underlying disks, hence don't record badblocks or retry - * in this case. + * bio with REQ_RAHEAD can fail at anytime, before such IO is submitted to the + * underlying disks, hence don't record badblocks or retry in this case. * * BLK_STS_INVAL means the bio was not valid for the underlying device. This * is a user error, not a device failure, so retrying or recording bad blocks @@ -300,6 +299,5 @@ static inline bool raid1_should_read_first(struct mddev *mddev, */ static inline bool raid1_should_handle_error(struct bio *bio) { - return !(bio->bi_opf & (REQ_RAHEAD | REQ_NOWAIT)) && - bio->bi_status != BLK_STS_INVAL; + return !(bio->bi_opf & REQ_RAHEAD) && bio->bi_status != BLK_STS_INVAL; } diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index afe2ca96ad8c..f0646fb24371 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -604,9 +604,9 @@ static void update_read_sectors(struct r1conf *conf, int disk, struct raid1_info *info = &conf->mirrors[disk]; atomic_inc(&info->rdev->nr_pending); - if (info->next_seq_sect != this_sector) - info->seq_start = this_sector; - info->next_seq_sect = this_sector + len; + if (READ_ONCE(info->next_seq_sect) != this_sector) + WRITE_ONCE(info->seq_start, this_sector); + WRITE_ONCE(info->next_seq_sect, this_sector + len); } static int choose_first_rdev(struct r1conf *conf, struct r1bio *r1_bio, @@ -735,8 +735,7 @@ static int choose_slow_rdev(struct r1conf *conf, struct r1bio *r1_bio, static bool is_sequential(struct r1conf *conf, int disk, struct r1bio *r1_bio) { - /* TODO: address issues with this check and concurrency. */ - return conf->mirrors[disk].next_seq_sect == r1_bio->sector || + return READ_ONCE(conf->mirrors[disk].next_seq_sect) == r1_bio->sector || READ_ONCE(conf->mirrors[disk].head_position) == r1_bio->sector; } @@ -747,15 +746,18 @@ static bool is_sequential(struct r1conf *conf, int disk, struct r1bio *r1_bio) static bool should_choose_next(struct r1conf *conf, int disk) { struct raid1_info *mirror = &conf->mirrors[disk]; + sector_t seq_start, next_seq_sect; int opt_iosize; if (!test_bit(Nonrot, &mirror->rdev->flags)) return false; opt_iosize = bdev_io_opt(mirror->rdev->bdev) >> 9; - return opt_iosize > 0 && mirror->seq_start != MaxSector && - mirror->next_seq_sect > opt_iosize && - mirror->next_seq_sect - opt_iosize >= mirror->seq_start; + seq_start = READ_ONCE(mirror->seq_start); + next_seq_sect = READ_ONCE(mirror->next_seq_sect); + return opt_iosize > 0 && seq_start != MaxSector && + next_seq_sect > opt_iosize && + next_seq_sect - opt_iosize >= seq_start; } static bool rdev_readable(struct md_rdev *rdev, struct r1bio *r1_bio) @@ -1051,10 +1053,8 @@ static void lower_barrier(struct r1conf *conf, sector_t sector_nr) wake_up(&conf->wait_barrier); } -static bool _wait_barrier(struct r1conf *conf, int idx, bool nowait) +static void _wait_barrier(struct r1conf *conf, int idx) { - bool ret = true; - /* * We need to increase conf->nr_pending[idx] very early here, * then raise_barrier() can be blocked when it waits for @@ -1085,7 +1085,7 @@ static bool _wait_barrier(struct r1conf *conf, int idx, bool nowait) */ if (!READ_ONCE(conf->array_frozen) && !atomic_read(&conf->barrier[idx])) - return ret; + return; /* * After holding conf->resync_lock, conf->nr_pending[idx] @@ -1104,26 +1104,18 @@ static bool _wait_barrier(struct r1conf *conf, int idx, bool nowait) wake_up_barrier(conf); /* Wait for the barrier in same barrier unit bucket to drop. */ - /* Return false when nowait flag is set */ - if (nowait) { - ret = false; - } else { - wait_event_lock_irq(conf->wait_barrier, - !conf->array_frozen && - !atomic_read(&conf->barrier[idx]), - conf->resync_lock); - atomic_inc(&conf->nr_pending[idx]); - } + wait_event_lock_irq(conf->wait_barrier, !conf->array_frozen && + !atomic_read(&conf->barrier[idx]), + conf->resync_lock); + atomic_inc(&conf->nr_pending[idx]); atomic_dec(&conf->nr_waiting[idx]); spin_unlock_irq(&conf->resync_lock); - return ret; } -static bool wait_read_barrier(struct r1conf *conf, sector_t sector_nr, bool nowait) +static void wait_read_barrier(struct r1conf *conf, sector_t sector_nr) { int idx = sector_to_idx(sector_nr); - bool ret = true; /* * Very similar to _wait_barrier(). The difference is, for read @@ -1135,7 +1127,7 @@ static bool wait_read_barrier(struct r1conf *conf, sector_t sector_nr, bool nowa atomic_inc(&conf->nr_pending[idx]); if (!READ_ONCE(conf->array_frozen)) - return ret; + return; spin_lock_irq(&conf->resync_lock); atomic_inc(&conf->nr_waiting[idx]); @@ -1147,27 +1139,19 @@ static bool wait_read_barrier(struct r1conf *conf, sector_t sector_nr, bool nowa wake_up_barrier(conf); /* Wait for array to be unfrozen */ - /* Return false when nowait flag is set */ - if (nowait) { - /* Return false when nowait flag is set */ - ret = false; - } else { - wait_event_lock_irq(conf->wait_barrier, - !conf->array_frozen, - conf->resync_lock); - atomic_inc(&conf->nr_pending[idx]); - } + wait_event_lock_irq(conf->wait_barrier, !conf->array_frozen, + conf->resync_lock); + atomic_inc(&conf->nr_pending[idx]); atomic_dec(&conf->nr_waiting[idx]); spin_unlock_irq(&conf->resync_lock); - return ret; } -static bool wait_barrier(struct r1conf *conf, sector_t sector_nr, bool nowait) +static void wait_barrier(struct r1conf *conf, sector_t sector_nr) { int idx = sector_to_idx(sector_nr); - return _wait_barrier(conf, idx, nowait); + _wait_barrier(conf, idx); } static void _allow_barrier(struct r1conf *conf, int idx) @@ -1342,7 +1326,6 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, int max_sectors; int rdisk; bool r1bio_existed = !!r1_bio; - bool nowait = bio->bi_opf & REQ_NOWAIT; /* * An md cloned bio indicates we are in the error path. @@ -1362,16 +1345,7 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, * Still need barrier for READ in case that whole * array is frozen. */ - if (!wait_read_barrier(conf, bio->bi_iter.bi_sector, nowait)) { - bio_wouldblock_error(bio); - - if (r1bio_existed) { - set_bit(R1BIO_Returned, &r1_bio->state); - raid_end_bio_io(r1_bio); - } - - return; - } + wait_read_barrier(conf, bio->bi_iter.bi_sector); if (!r1_bio) r1_bio = alloc_r1bio(mddev, bio); @@ -1406,14 +1380,10 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, md_bitmap_enabled(mddev, false)) { /* * Reading from a write-mostly device must take care not to - * over-take any writes that are 'behind' - */ - mddev_add_trace_msg(mddev, "raid1 wait behind writes"); - if (!mddev->bitmap_ops->wait_behind_writes(mddev, nowait)) { - bio_wouldblock_error(bio); - set_bit(R1BIO_Returned, &r1_bio->state); - goto err_handle; - } + * over-take any writes that are 'behind' + */ + mddev_add_trace_msg(mddev, "raid1 wait behind writes"); + mddev->bitmap_ops->wait_behind_writes(mddev); } if (max_sectors < bio_sectors(bio)) { @@ -1435,7 +1405,6 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, } read_bio = bio_alloc_clone(mirror->rdev->bdev, bio, gfp, &mddev->bio_set); - read_bio->bi_opf &= ~REQ_NOWAIT; r1_bio->bios[rdisk] = read_bio; read_bio->bi_iter.bi_sector = r1_bio->sector + @@ -1454,7 +1423,7 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, raid_end_bio_io(r1_bio); } -static bool wait_blocked_rdev(struct mddev *mddev, struct bio *bio) +static void wait_blocked_rdev(struct mddev *mddev, struct bio *bio) { struct r1conf *conf = mddev->private; int disks = conf->raid_disks * 2; @@ -1474,9 +1443,6 @@ static bool wait_blocked_rdev(struct mddev *mddev, struct bio *bio) set_bit(BlockedBadBlocks, &rdev->flags); if (rdev_blocked(rdev)) { - if (bio->bi_opf & REQ_NOWAIT) - return false; - mddev_add_trace_msg(rdev->mddev, "raid1 wait rdev %d blocked", rdev->raid_disk); atomic_inc(&rdev->nr_pending); @@ -1484,8 +1450,6 @@ static bool wait_blocked_rdev(struct mddev *mddev, struct bio *bio) goto retry; } } - - return true; } static void raid1_start_write_behind(struct mddev *mddev, struct r1bio *r1_bio, @@ -1521,18 +1485,13 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, unsigned long flags; int first_clone; bool write_behind = false; - bool nowait = bio->bi_opf & REQ_NOWAIT; + bool atomic = bio->bi_opf & REQ_ATOMIC; bool is_discard = op_is_discard(bio->bi_opf); sector_t sector = bio->bi_iter.bi_sector; if (mddev_is_clustered(mddev) && mddev->cluster_ops->area_resyncing(mddev, WRITE, sector, bio_end_sector(bio))) { - - if (nowait) { - bio_wouldblock_error(bio); - return false; - } wait_event_idle(conf->wait_barrier, !mddev->cluster_ops->area_resyncing(mddev, WRITE, sector, @@ -1544,15 +1503,9 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, * thread has put up a bar for new requests. * Continue immediately if no resync is active currently. */ - if (!wait_barrier(conf, sector, nowait)) { - bio_wouldblock_error(bio); - return false; - } + wait_barrier(conf, sector); - if (!wait_blocked_rdev(mddev, bio)) { - bio_wouldblock_error(bio); - goto err_allow_barrier; - } + wait_blocked_rdev(mddev, bio); r1_bio = alloc_r1bio(mddev, bio); r1_bio->sectors = max_sectors; @@ -1579,6 +1532,8 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, */ if (!is_discard && rdev && test_bit(WriteMostly, &rdev->flags)) write_behind = true; + if (atomic && max_sectors > BIO_MAX_VECS * (PAGE_SIZE >> 9)) + write_behind = false; r1_bio->bios[i] = NULL; if (!rdev || test_bit(Faulty, &rdev->flags)) @@ -1604,19 +1559,6 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, if (is_bad) { int good_sectors; - /* - * We cannot atomically write this, so just - * error in that case. It could be possible to - * atomically write other mirrors, but the - * complexity of supporting that is not worth - * the benefit. - */ - if (bio->bi_opf & REQ_ATOMIC) { - bio->bi_status = BLK_STS_NOTSUPP; - bio_endio(bio); - goto err_dec_pending; - } - good_sectors = first_bad - sector; if (good_sectors < max_sectors) max_sectors = good_sectors; @@ -1637,6 +1579,11 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, max_sectors = min_t(int, max_sectors, BIO_MAX_VECS * (PAGE_SIZE >> 9)); if (max_sectors < bio_sectors(bio)) { + if (atomic) { + bio_io_error(bio); + goto err_dec_pending; + } + bio = bio_submit_split_bioset(bio, max_sectors, &conf->bio_split); if (!bio) @@ -1681,7 +1628,6 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, wait_for_serialization(rdev, r1_bio); } - mbio->bi_opf &= ~REQ_NOWAIT; r1_bio->bios[i] = mbio; mbio->bi_iter.bi_sector = sector + rdev->data_offset; @@ -1720,8 +1666,6 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, } free_r1bio(r1_bio); - -err_allow_barrier: allow_barrier(conf, sector); return false; @@ -1850,7 +1794,7 @@ static void close_sync(struct r1conf *conf) int idx; for (idx = 0; idx < BARRIER_BUCKETS_NR; idx++) { - _wait_barrier(conf, idx, false); + _wait_barrier(conf, idx); _allow_barrier(conf, idx); } @@ -3228,6 +3172,7 @@ static int raid1_set_limits(struct mddev *mddev) md_init_stacking_limits(&lim); lim.max_write_zeroes_sectors = 0; lim.max_hw_wzeroes_unmap_sectors = 0; + lim.chunk_sectors = BARRIER_UNIT_SECTOR_SIZE; lim.logical_block_size = mddev->logical_block_size; lim.features |= BLK_FEAT_ATOMIC_WRITES; lim.features |= BLK_FEAT_PCI_P2PDMA; @@ -3485,8 +3430,6 @@ static void *raid1_takeover(struct mddev *mddev) mddev->new_chunk_sectors = 0; conf = setup_conf(mddev); if (!IS_ERR(conf)) { - /* Array must appear to be quiesced */ - conf->array_frozen = 1; mddev_clear_unsupported_flags(mddev, UNSUPPORTED_MDDEV_FLAGS); } diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index 0a3cfdd3f5df..1093c798d9dd 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -103,13 +103,23 @@ static inline struct r10bio *get_resync_r10bio(struct bio *bio) return get_resync_pages(bio)->raid_bio; } -static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data) +static inline int calc_r10bio_size(unsigned int raid_disks) { - struct r10conf *conf = data; - int size = offsetof(struct r10bio, devs[conf->geo.raid_disks]); + return offsetof(struct r10bio, devs[raid_disks]); +} - /* allocate a r10bio with room for raid_disks entries in the - * bios array */ +static mempool_t *create_r10bio_pool(unsigned int raid_disks) +{ + int size = calc_r10bio_size(raid_disks); + + return mempool_create_kmalloc_pool(NR_RAID_BIOS, size); +} + +static struct r10bio *alloc_r10bio(unsigned int raid_disks, gfp_t gfp_flags) +{ + int size = calc_r10bio_size(raid_disks); + + /* allocate a r10bio sized for current geometry */ return kzalloc(size, gfp_flags); } @@ -137,7 +147,7 @@ static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data) int nalloc, nalloc_rp; struct resync_pages *rps; - r10_bio = r10bio_pool_alloc(gfp_flags, conf); + r10_bio = alloc_r10bio(conf->geo.raid_disks, gfp_flags); if (!r10_bio) return NULL; @@ -277,7 +287,7 @@ static void free_r10bio(struct r10bio *r10_bio) struct r10conf *conf = r10_bio->mddev->private; put_all_bios(conf, r10_bio); - mempool_free(r10_bio, &conf->r10bio_pool); + mempool_free(r10_bio, conf->r10bio_pool); } static void put_buf(struct r10bio *r10_bio) @@ -321,20 +331,24 @@ static void raid_end_bio_io(struct r10bio *r10_bio) { struct bio *bio = r10_bio->master_bio; struct r10conf *conf = r10_bio->mddev->private; + bool returned = true; if (!test_and_set_bit(R10BIO_Returned, &r10_bio->state)) { if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) bio->bi_status = BLK_STS_IOERR; - bio_endio(bio); + returned = false; } + free_r10bio(r10_bio); + + if (!returned) + bio_endio(bio); + /* * Wake up any possible resync thread that waits for the device * to go idle. */ allow_barrier(conf); - - free_r10bio(r10_bio); } /* @@ -1002,32 +1016,22 @@ static bool wait_barrier_nolock(struct r10conf *conf) return false; } -static bool wait_barrier(struct r10conf *conf, bool nowait) +static void wait_barrier(struct r10conf *conf) { - bool ret = true; - if (wait_barrier_nolock(conf)) - return true; + return; write_seqlock_irq(&conf->resync_lock); if (conf->barrier) { - /* Return false when nowait flag is set */ - if (nowait) { - ret = false; - } else { - conf->nr_waiting++; - mddev_add_trace_msg(conf->mddev, "raid10 wait barrier"); - wait_event_barrier(conf, stop_waiting_barrier(conf)); - conf->nr_waiting--; - } + conf->nr_waiting++; + mddev_add_trace_msg(conf->mddev, "raid10 wait barrier"); + wait_event_barrier(conf, stop_waiting_barrier(conf)); + conf->nr_waiting--; if (!conf->nr_waiting) wake_up(&conf->wait_barrier); } - /* Only increment nr_pending when we wait */ - if (ret) - atomic_inc(&conf->nr_pending); + atomic_inc(&conf->nr_pending); write_sequnlock_irq(&conf->resync_lock); - return ret; } static void allow_barrier(struct r10conf *conf) @@ -1119,30 +1123,22 @@ static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule) * currently. * 2. If IO spans the reshape position. Need to wait for reshape to pass. */ -static bool regular_request_wait(struct mddev *mddev, struct r10conf *conf, +static void regular_request_wait(struct mddev *mddev, struct r10conf *conf, struct bio *bio, sector_t sectors) { - /* Bail out if REQ_NOWAIT is set for the bio */ - if (!wait_barrier(conf, bio->bi_opf & REQ_NOWAIT)) { - bio_wouldblock_error(bio); - return false; - } + wait_barrier(conf); + while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && bio->bi_iter.bi_sector < conf->reshape_progress && bio->bi_iter.bi_sector + sectors > conf->reshape_progress) { allow_barrier(conf); - if (bio->bi_opf & REQ_NOWAIT) { - bio_wouldblock_error(bio); - return false; - } mddev_add_trace_msg(conf->mddev, "raid10 wait reshape"); wait_event(conf->wait_barrier, conf->reshape_progress <= bio->bi_iter.bi_sector || conf->reshape_progress >= bio->bi_iter.bi_sector + sectors); - wait_barrier(conf, false); + wait_barrier(conf); } - return true; } static void raid10_read_request(struct mddev *mddev, struct bio *bio, @@ -1191,10 +1187,7 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, } } - if (!regular_request_wait(mddev, conf, bio, r10_bio->sectors)) { - free_r10bio(r10_bio); - return; - } + regular_request_wait(mddev, conf, bio, r10_bio->sectors); rdev = read_balance(conf, r10_bio, &max_sectors); if (!rdev) { @@ -1215,7 +1208,7 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, allow_barrier(conf); bio = bio_submit_split_bioset(bio, max_sectors, &conf->bio_split); - wait_barrier(conf, false); + wait_barrier(conf); if (!bio) { set_bit(R10BIO_Returned, &r10_bio->state); goto err_handle; @@ -1231,7 +1224,6 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, r10_bio->master_bio = bio; } read_bio = bio_alloc_clone(rdev->bdev, bio, gfp, &mddev->bio_set); - read_bio->bi_opf &= ~REQ_NOWAIT; r10_bio->devs[slot].bio = read_bio; r10_bio->devs[slot].rdev = rdev; @@ -1265,7 +1257,6 @@ static void raid10_write_one_disk(struct mddev *mddev, struct r10bio *r10_bio, conf->mirrors[devnum].rdev; mbio = bio_alloc_clone(rdev->bdev, bio, GFP_NOIO, &mddev->bio_set); - mbio->bi_opf &= ~REQ_NOWAIT; if (replacement) r10_bio->devs[n_copy].repl_bio = mbio; else @@ -1344,7 +1335,7 @@ static void wait_blocked_dev(struct mddev *mddev, struct r10bio *r10_bio) "raid10 %s wait rdev %d blocked", __func__, blocked_rdev->raid_disk); md_wait_for_blocked_rdev(blocked_rdev, mddev); - wait_barrier(conf, false); + wait_barrier(conf); goto retry_wait; } } @@ -1356,33 +1347,20 @@ static bool raid10_write_request(struct mddev *mddev, struct bio *bio, int i, k; sector_t sectors; int max_sectors; + bool atomic = bio->bi_opf & REQ_ATOMIC; if ((mddev_is_clustered(mddev) && mddev->cluster_ops->area_resyncing(mddev, WRITE, bio->bi_iter.bi_sector, bio_end_sector(bio)))) { - DEFINE_WAIT(w); - /* Bail out if REQ_NOWAIT is set for the bio */ - if (bio->bi_opf & REQ_NOWAIT) { - bio_wouldblock_error(bio); - return false; - } - for (;;) { - prepare_to_wait(&conf->wait_barrier, - &w, TASK_IDLE); - if (!mddev->cluster_ops->area_resyncing(mddev, WRITE, - bio->bi_iter.bi_sector, bio_end_sector(bio))) - break; - schedule(); - } - finish_wait(&conf->wait_barrier, &w); + wait_event_idle(conf->wait_barrier, + !mddev->cluster_ops->area_resyncing(mddev, WRITE, + bio->bi_iter.bi_sector, + bio_end_sector(bio))); } sectors = r10_bio->sectors; - if (!regular_request_wait(mddev, conf, bio, sectors)) { - free_r10bio(r10_bio); - return false; - } + regular_request_wait(mddev, conf, bio, sectors); if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && (mddev->reshape_backwards @@ -1395,11 +1373,6 @@ static bool raid10_write_request(struct mddev *mddev, struct bio *bio, set_mask_bits(&mddev->sb_flags, 0, BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING)); md_wakeup_thread(mddev->thread); - if (bio->bi_opf & REQ_NOWAIT) { - allow_barrier(conf); - bio_wouldblock_error(bio); - return false; - } mddev_add_trace_msg(conf->mddev, "raid10 wait reshape metadata"); wait_event(mddev->sb_wait, @@ -1462,16 +1435,6 @@ static bool raid10_write_request(struct mddev *mddev, struct bio *bio, if (is_bad) { int good_sectors; - /* - * We cannot atomically write this, so just - * error in that case. It could be possible to - * atomically write other mirrors, but the - * complexity of supporting that is not worth - * the benefit. - */ - if (bio->bi_opf & REQ_ATOMIC) - goto err_handle; - good_sectors = first_bad - dev_sector; if (good_sectors < max_sectors) max_sectors = good_sectors; @@ -1491,10 +1454,11 @@ static bool raid10_write_request(struct mddev *mddev, struct bio *bio, r10_bio->sectors = max_sectors; if (r10_bio->sectors < bio_sectors(bio)) { - allow_barrier(conf); + if (atomic) + goto err_handle; + bio = bio_submit_split_bioset(bio, r10_bio->sectors, &conf->bio_split); - wait_barrier(conf, false); if (!bio) { set_bit(R10BIO_Returned, &r10_bio->state); goto err_handle; @@ -1542,7 +1506,7 @@ static bool __make_request(struct mddev *mddev, struct bio *bio, int sectors) struct r10conf *conf = mddev->private; struct r10bio *r10_bio; - r10_bio = mempool_alloc(&conf->r10bio_pool, GFP_NOIO); + r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO); r10_bio->master_bio = bio; r10_bio->sectors = sectors; @@ -1577,9 +1541,11 @@ static void raid_end_discard_bio(struct r10bio *r10bio) free_r10bio(r10bio); r10bio = first_r10bio; } else { + struct bio *master_bio = r10bio->master_bio; + md_write_end(r10bio->mddev); - bio_endio(r10bio->master_bio); free_r10bio(r10bio); + bio_endio(master_bio); break; } } @@ -1637,11 +1603,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) return -EAGAIN; - if (!wait_barrier(conf, bio->bi_opf & REQ_NOWAIT)) { - bio_wouldblock_error(bio); - md_write_end(mddev); - return 0; - } + wait_barrier(conf); /* * Check reshape again to avoid reshape happens after checking @@ -1692,7 +1654,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) allow_barrier(conf); /* Resend the fist split part */ submit_bio_noacct(split); - wait_barrier(conf, false); + wait_barrier(conf); } div_u64_rem(bio_end, stripe_size, &remainder); if (remainder) { @@ -1712,7 +1674,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) /* Resend the second split part */ submit_bio_noacct(bio); bio = split; - wait_barrier(conf, false); + wait_barrier(conf); } bio_start = bio->bi_iter.bi_sector; @@ -1742,7 +1704,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) (last_stripe_index << geo->chunk_shift); retry_discard: - r10_bio = mempool_alloc(&conf->r10bio_pool, GFP_NOIO); + r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO); r10_bio->mddev = mddev; r10_bio->state = 0; r10_bio->sectors = 0; @@ -1870,7 +1832,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) end_disk_offset += geo->stride; atomic_inc(&first_r10bio->remaining); raid_end_discard_bio(r10_bio); - wait_barrier(conf, false); + wait_barrier(conf); goto retry_discard; } @@ -1886,6 +1848,7 @@ static bool raid10_make_request(struct mddev *mddev, struct bio *bio) { struct r10conf *conf = mddev->private; sector_t chunk_mask = (conf->geo.chunk_mask & conf->prev.chunk_mask); + const int rw = bio_data_dir(bio); int chunk_sects = chunk_mask + 1; int sectors = bio_sectors(bio); @@ -1911,6 +1874,15 @@ static bool raid10_make_request(struct mddev *mddev, struct bio *bio) sectors = chunk_sects - (bio->bi_iter.bi_sector & (chunk_sects - 1)); + + bio = mddev_bio_split_at_reshape_offset(mddev, bio, §ors, + &conf->bio_split); + if (!bio) { + if (rw == WRITE) + md_write_end(mddev); + return true; + } + if (!__make_request(mddev, bio, sectors)) md_write_end(mddev); @@ -2069,7 +2041,7 @@ static void print_conf(struct r10conf *conf) static void close_sync(struct r10conf *conf) { - wait_barrier(conf, false); + wait_barrier(conf); allow_barrier(conf); mempool_exit(&conf->r10buf_pool); @@ -3365,7 +3337,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, struct md_rdev *rdev = conf->mirrors[j].rdev; if (rdev == NULL || test_bit(Faulty, &rdev->flags)) { - still_degraded = false; + still_degraded = true; break; } } @@ -3844,7 +3816,7 @@ static void raid10_free_conf(struct r10conf *conf) if (!conf) return; - mempool_exit(&conf->r10bio_pool); + mempool_destroy(conf->r10bio_pool); kfree(conf->mirrors); kfree(conf->mirrors_old); kfree(conf->mirrors_new); @@ -3891,9 +3863,8 @@ static struct r10conf *setup_conf(struct mddev *mddev) conf->geo = geo; conf->copies = copies; - err = mempool_init(&conf->r10bio_pool, NR_RAID_BIOS, r10bio_pool_alloc, - rbio_pool_free, conf); - if (err) + conf->r10bio_pool = create_r10bio_pool(conf->geo.raid_disks); + if (!conf->r10bio_pool) goto out; err = bioset_init(&conf->bio_split, BIO_POOL_SIZE, 0, 0); @@ -4284,6 +4255,10 @@ static int raid10_check_reshape(struct mddev *mddev) if (conf->geo.far_copies != 1 && !conf->geo.far_offset) return -EINVAL; + if (mddev->bitmap_id == ID_LLBITMAP && + mddev->new_chunk_sectors && + mddev->new_chunk_sectors < mddev->chunk_sectors) + return -EOPNOTSUPP; if (setup_geo(&geo, mddev, geo_start) != conf->copies) /* mustn't change number of copies */ @@ -4387,9 +4362,16 @@ static int raid10_start_reshape(struct mddev *mddev) struct md_rdev *rdev; int spares = 0; int ret; + mempool_t *new_pool = NULL; if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) return -EBUSY; + if (md_bitmap_enabled(mddev, false) && + mddev->bitmap_ops->reshape_can_start) { + ret = mddev->bitmap_ops->reshape_can_start(mddev); + if (ret) + return ret; + } if (setup_geo(&new, mddev, geo_start) != conf->copies) return -EINVAL; @@ -4423,6 +4405,11 @@ static int raid10_start_reshape(struct mddev *mddev) return -EINVAL; conf->offset_diff = min_offset_diff; + if (mddev->delta_disks > 0) { + new_pool = create_r10bio_pool(new.raid_disks); + if (!new_pool) + return -ENOMEM; + } spin_lock_irq(&conf->device_lock); if (conf->mirrors_new) { memcpy(conf->mirrors_new, conf->mirrors, @@ -4523,6 +4510,10 @@ static int raid10_start_reshape(struct mddev *mddev) mddev->raid_disks = conf->geo.raid_disks; mddev->reshape_position = conf->reshape_progress; set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); + if (new_pool) { + mempool_destroy(conf->r10bio_pool); + conf->r10bio_pool = new_pool; + } clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); @@ -4545,6 +4536,7 @@ static int raid10_start_reshape(struct mddev *mddev) conf->reshape_safe = MaxSector; mddev->reshape_position = MaxSector; spin_unlock_irq(&conf->device_lock); + mempool_destroy(new_pool); return ret; } @@ -4702,7 +4694,14 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, if (need_flush || time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) { /* Need to update reshape_position in metadata */ - wait_barrier(conf, false); + wait_barrier(conf); + if (md_bitmap_enabled(mddev, false) && + mddev->bitmap_ops->reshape_mark && + conf->reshape_safe != conf->reshape_progress) { + mddev->bitmap_ops->reshape_mark(mddev, conf->reshape_safe, + conf->reshape_progress); + mddev->bitmap_ops->unplug(mddev, true); + } mddev->reshape_position = conf->reshape_progress; if (mddev->reshape_backwards) mddev->curr_resync_completed = raid10_size(mddev, 0, 0) @@ -4901,9 +4900,19 @@ static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio) static void end_reshape(struct r10conf *conf) { + struct mddev *mddev = conf->mddev; + if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) return; + if (md_bitmap_enabled(mddev, false) && + mddev->bitmap_ops->reshape_mark && + conf->reshape_safe != conf->reshape_progress) { + mddev->bitmap_ops->reshape_mark(mddev, conf->reshape_safe, + conf->reshape_progress); + mddev->bitmap_ops->unplug(mddev, true); + } + spin_lock_irq(&conf->device_lock); conf->prev = conf->geo; md_finish_reshape(conf->mddev); @@ -5035,10 +5044,15 @@ static void end_reshape_request(struct r10bio *r10_bio) static void raid10_finish_reshape(struct mddev *mddev) { struct r10conf *conf = mddev->private; + bool llbitmap = mddev->bitmap_id == ID_LLBITMAP && + md_bitmap_enabled(mddev, false); if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) return; + if (llbitmap && mddev->bitmap_ops->reshape_finish) + mddev->bitmap_ops->reshape_finish(mddev); + if (mddev->delta_disks > 0) { if (mddev->resync_offset > mddev->resync_max_sectors) { mddev->resync_offset = mddev->resync_max_sectors; @@ -5065,6 +5079,15 @@ static void raid10_finish_reshape(struct mddev *mddev) mddev->reshape_backwards = 0; } +static sector_t raid10_bitmap_sync_size(struct mddev *mddev, bool previous) +{ + struct r10conf *conf = mddev->private; + + if (previous) + return raid10_size(mddev, 0, 0); + return raid10_size(mddev, 0, conf->geo.raid_disks); +} + static struct md_personality raid10_personality = { .head = { @@ -5091,6 +5114,8 @@ static struct md_personality raid10_personality = .start_reshape = raid10_start_reshape, .finish_reshape = raid10_finish_reshape, .update_reshape_pos = raid10_update_reshape_pos, + .bitmap_sync_size = raid10_bitmap_sync_size, + .bitmap_array_sectors = raid10_bitmap_sync_size, }; static int __init raid10_init(void) diff --git a/drivers/md/raid10.h b/drivers/md/raid10.h index ec79d87fb92f..b711626a5db7 100644 --- a/drivers/md/raid10.h +++ b/drivers/md/raid10.h @@ -87,7 +87,7 @@ struct r10conf { */ wait_queue_head_t wait_barrier; - mempool_t r10bio_pool; + mempool_t *r10bio_pool; mempool_t r10buf_pool; struct page *tmppage; struct bio_set bio_split; diff --git a/drivers/md/raid5-ppl.c b/drivers/md/raid5-ppl.c index 7be1648c4e4f..7f8a9d3fd578 100644 --- a/drivers/md/raid5-ppl.c +++ b/drivers/md/raid5-ppl.c @@ -643,8 +643,10 @@ static void ppl_do_flush(struct ppl_io_unit *io) log->disk_flush_bitmap = 0; for (i = flushed_disks ; i < raid_disks; i++) { - if (atomic_dec_and_test(&io->pending_flushes)) + if (atomic_dec_and_test(&io->pending_flushes)) { ppl_io_unit_finished(io); + break; + } } } diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index ffb5fcde54a9..b91545ce090d 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -801,7 +801,7 @@ static bool is_inactive_blocked(struct r5conf *conf, int hash) return true; return (atomic_read(&conf->active_stripes) < - (conf->max_nr_stripes * 3 / 4)); + (READ_ONCE(conf->max_nr_stripes) * 3 / 4)); } struct stripe_head *raid5_get_active_stripe(struct r5conf *conf, @@ -2471,11 +2471,6 @@ static int scribble_alloc(struct raid5_percpu *percpu, sizeof(unsigned int) * (num + 2); void *scribble; - /* - * If here is in raid array suspend context, it is in memalloc noio - * context as well, there is no potential recursive memory reclaim - * I/Os with the GFP_KERNEL flag. - */ scribble = kvmalloc_array(cnt, obj_size, GFP_KERNEL); if (!scribble) return -ENOMEM; @@ -2490,6 +2485,7 @@ static int scribble_alloc(struct raid5_percpu *percpu, static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors) { unsigned long cpu; + unsigned int noio_flags; int err = 0; /* Never shrink. */ @@ -2498,6 +2494,7 @@ static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors) return 0; raid5_quiesce(conf->mddev, true); + noio_flags = memalloc_noio_save(); cpus_read_lock(); for_each_present_cpu(cpu) { @@ -2511,6 +2508,7 @@ static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors) } cpus_read_unlock(); + memalloc_noio_restore(noio_flags); raid5_quiesce(conf->mddev, false); if (!err) { @@ -2785,6 +2783,7 @@ static void raid5_end_read_request(struct bio * bi) } else { int retry = 0; int set_bad = 0; + int max_nr_stripes = READ_ONCE(conf->max_nr_stripes); clear_bit(R5_UPTODATE, &sh->dev[i].flags); if (!(bi->bi_status == BLK_STS_PROTECTION)) @@ -2810,13 +2809,12 @@ static void raid5_end_read_request(struct bio * bi) mdname(conf->mddev), (unsigned long long)s, rdev->bdev); - } else if (atomic_read(&rdev->read_errors) - > conf->max_nr_stripes) { + } else if (atomic_read(&rdev->read_errors) > max_nr_stripes) { if (!test_bit(Faulty, &rdev->flags)) { pr_warn("md/raid:%s: %d read_errors > %d stripes\n", mdname(conf->mddev), atomic_read(&rdev->read_errors), - conf->max_nr_stripes); + max_nr_stripes); pr_warn("md/raid:%s: Too many read errors, failing device %pg.\n", mdname(conf->mddev), rdev->bdev); } @@ -3553,7 +3551,7 @@ static void __add_stripe_bio(struct stripe_head *sh, struct bio *bi, sh->dev[dd_idx].sector); if (conf->mddev->bitmap && firstwrite && !sh->batch_head) { - sh->bm_seq = conf->seq_flush+1; + sh->bm_seq = READ_ONCE(conf->seq_flush) + 1; set_bit(STRIPE_BIT_DELAY, &sh->state); } } @@ -3745,17 +3743,89 @@ handle_failed_sync(struct r5conf *conf, struct stripe_head *sh, md_sync_error(conf->mddev); } +/* + * handle_failed_reshape - handle failed stripes when reshape failed and + * degraded devices >= max_degraded + * + * handle following kinds of stripe: + * 1. cleanup the following kinds of destination stripe: + * - new regions past the old end of the array, zero-filled in place, + * requires no source data. + * (STRIPE_EXPANDING | STRIPE_EXPAND_READY) + * - prepare source data chunks already done, and writeout failed + * (STRIPE_EXPAND_READY) + * 2. dest stripes that need source data (STRIPE_EXPANDING, no STRIPE_HANDLE) + * - these kind of stripes sit idle in the stripe cache and are never seen + * by handle_stripe(). So clean up indirectly when their source stripe + * (type 3) is processed. + * 3. src stripes (STRIPE_EXPAND_SOURCE) + * - hit handle_stripe() after their member disks are marked Faulty. + * - clear STRIPE_EXPAND_SOURCE, finds and cleanup all dependent destination + * stripes that were waiting for data. + * - walks the source's data disks, compute the corresponding destination + * sector, looks up the destination stripe, and do cleanup(clear flags, + * dec counters, call md_done_sync()) + */ +static void handle_failed_reshape(struct r5conf *conf, struct stripe_head *sh, + struct stripe_head_state *s) +{ + int i; + bool was_expanding = test_and_clear_bit(STRIPE_EXPANDING, &sh->state); + bool was_ready = test_and_clear_bit(STRIPE_EXPAND_READY, &sh->state); + + if (was_expanding || was_ready) { + atomic_dec(&conf->reshape_stripes); + wake_up(&conf->wait_for_reshape); + md_done_sync(conf->mddev, RAID5_STRIPE_SECTORS(conf)); + } + + s->expanded = 0; + s->expanding = 0; + + /* release the destination stripes that are waiting to be filled */ + if (test_and_clear_bit(STRIPE_EXPAND_SOURCE, &sh->state)) { + for (i = 0; i < sh->disks; i++) { + int dd_idx; + struct stripe_head *sh2; + sector_t bn, sec; + + if (i == sh->pd_idx) + continue; + if (conf->level == 6 && i == sh->qd_idx) + continue; + + bn = raid5_compute_blocknr(sh, i, 1); + sec = raid5_compute_sector(conf, bn, 0, &dd_idx, NULL); + sh2 = raid5_get_active_stripe(conf, NULL, sec, + R5_GAS_NOBLOCK | + R5_GAS_NOQUIESCE); + if (!sh2) + continue; + + if (test_and_clear_bit(STRIPE_EXPANDING, &sh2->state)) { + atomic_dec(&conf->reshape_stripes); + wake_up(&conf->wait_for_reshape); + md_done_sync(conf->mddev, + RAID5_STRIPE_SECTORS(conf)); + } + + clear_bit(STRIPE_EXPAND_READY, &sh2->state); + + raid5_release_stripe(sh2); + } + } +} + static int want_replace(struct stripe_head *sh, int disk_idx) { struct md_rdev *rdev; int rv = 0; rdev = sh->raid_conf->disks[disk_idx].replacement; - if (rdev - && !test_bit(Faulty, &rdev->flags) - && !test_bit(In_sync, &rdev->flags) - && (rdev->recovery_offset <= sh->sector - || rdev->mddev->resync_offset <= sh->sector)) + if (rdev && !test_bit(Faulty, &rdev->flags) && + !test_bit(In_sync, &rdev->flags) && + (READ_ONCE(rdev->recovery_offset) <= sh->sector || + rdev->mddev->resync_offset <= sh->sector)) rv = 1; return rv; } @@ -4672,7 +4742,8 @@ static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s) */ rdev = conf->disks[i].replacement; if (rdev && !test_bit(Faulty, &rdev->flags) && - rdev->recovery_offset >= sh->sector + RAID5_STRIPE_SECTORS(conf) && + READ_ONCE(rdev->recovery_offset) >= + sh->sector + RAID5_STRIPE_SECTORS(conf) && !rdev_has_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf))) set_bit(R5_ReadRepl, &dev->flags); @@ -4714,7 +4785,7 @@ static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s) } else if (test_bit(In_sync, &rdev->flags)) set_bit(R5_Insync, &dev->flags); else if (sh->sector + RAID5_STRIPE_SECTORS(conf) <= - rdev->recovery_offset) { + READ_ONCE(rdev->recovery_offset)) { /* * in sync if: * - normal IO, or @@ -5025,6 +5096,8 @@ static void handle_stripe(struct stripe_head *sh) handle_failed_stripe(conf, sh, &s, disks); if (s.syncing + s.replacing) handle_failed_sync(conf, sh, &s); + if (s.expanding + s.expanded) + handle_failed_reshape(conf, sh, &s); } /* Now we check to see if any write operations have recently @@ -5458,13 +5531,13 @@ static int raid5_read_one_chunk(struct mddev *mddev, struct bio *raid_bio) rdev = conf->disks[dd_idx].replacement; if (!rdev || test_bit(Faulty, &rdev->flags) || - rdev->recovery_offset < end_sector) { + READ_ONCE(rdev->recovery_offset) < end_sector) { rdev = conf->disks[dd_idx].rdev; if (!rdev) return 0; if (test_bit(Faulty, &rdev->flags) || !(test_bit(In_sync, &rdev->flags) || - rdev->recovery_offset >= end_sector)) + READ_ONCE(rdev->recovery_offset) >= end_sector)) return 0; } @@ -5718,13 +5791,8 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi) struct bio *orig_bi = bi; int stripe_sectors; - /* We need to handle this when io_uring supports discard/trim */ - if (WARN_ON_ONCE(bi->bi_opf & REQ_NOWAIT)) - return; - if (mddev->reshape_position != MaxSector) - /* Skip discard while reshape is happening */ - return; + goto complete_again; if (!raid5_discard_limits(mddev, bi)) return; @@ -5799,7 +5867,7 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi) } spin_unlock_irq(&sh->stripe_lock); if (conf->mddev->bitmap) { - sh->bm_seq = conf->seq_flush + 1; + sh->bm_seq = READ_ONCE(conf->seq_flush) + 1; set_bit(STRIPE_BIT_DELAY, &sh->state); } @@ -5811,6 +5879,11 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi) } bio_endio(bi); + return; + +complete_again: + /* Skip discard while reshape is happening */ + bio_endio_status(bi, BLK_STS_AGAIN); } static bool ahead_of_reshape(struct mddev *mddev, sector_t sector, @@ -5942,6 +6015,34 @@ static enum reshape_loc get_reshape_loc(struct mddev *mddev, return LOC_BEHIND_RESHAPE; } +static void raid5_bitmap_sector_map(struct mddev *mddev, sector_t *offset, + unsigned long *sectors, + bool previous) +{ + struct r5conf *conf = mddev->private; + sector_t start = *offset; + sector_t end = start + *sectors; + int sectors_per_chunk; + int dd_idx; + + if (previous) + sectors_per_chunk = conf->prev_chunk_sectors * + (conf->previous_raid_disks - conf->max_degraded); + else + sectors_per_chunk = conf->chunk_sectors * + (conf->raid_disks - conf->max_degraded); + sector_div(start, sectors_per_chunk); + start *= sectors_per_chunk; + if (sector_div(end, sectors_per_chunk)) + end++; + end *= sectors_per_chunk; + + start = raid5_compute_sector(conf, start, previous, &dd_idx, NULL); + end = raid5_compute_sector(conf, end, previous, &dd_idx, NULL); + *offset = start; + *sectors = end - start; +} + static void raid5_bitmap_sector(struct mddev *mddev, sector_t *offset, unsigned long *sectors) { @@ -5949,18 +6050,11 @@ static void raid5_bitmap_sector(struct mddev *mddev, sector_t *offset, sector_t start = *offset; sector_t end = start + *sectors; sector_t prev_start = start; - sector_t prev_end = end; - int sectors_per_chunk; + unsigned long prev_sectors = end - start; enum reshape_loc loc; - int dd_idx; - sectors_per_chunk = conf->chunk_sectors * - (conf->raid_disks - conf->max_degraded); - start = round_down(start, sectors_per_chunk); - end = round_up(end, sectors_per_chunk); - - start = raid5_compute_sector(conf, start, 0, &dd_idx, NULL); - end = raid5_compute_sector(conf, end, 0, &dd_idx, NULL); + raid5_bitmap_sector_map(mddev, &start, sectors, false); + end = start + *sectors; /* * For LOC_INSIDE_RESHAPE, this IO will wait for reshape to make @@ -5969,17 +6063,10 @@ static void raid5_bitmap_sector(struct mddev *mddev, sector_t *offset, loc = get_reshape_loc(mddev, conf, prev_start); if (likely(loc != LOC_AHEAD_OF_RESHAPE)) { *offset = start; - *sectors = end - start; return; } - sectors_per_chunk = conf->prev_chunk_sectors * - (conf->previous_raid_disks - conf->max_degraded); - prev_start = round_down(prev_start, sectors_per_chunk); - prev_end = round_down(prev_end, sectors_per_chunk); - - prev_start = raid5_compute_sector(conf, prev_start, 1, &dd_idx, NULL); - prev_end = raid5_compute_sector(conf, prev_end, 1, &dd_idx, NULL); + raid5_bitmap_sector_map(mddev, &prev_start, &prev_sectors, true); /* * for LOC_AHEAD_OF_RESHAPE, reshape can make progress before this IO @@ -5987,7 +6074,7 @@ static void raid5_bitmap_sector(struct mddev *mddev, sector_t *offset, * we set bits for both. */ *offset = min(start, prev_start); - *sectors = max(end, prev_end) - *offset; + *sectors = max(end, prev_start + prev_sectors) - *offset; } static enum stripe_result make_stripe_request(struct mddev *mddev, @@ -6134,9 +6221,11 @@ static bool raid5_make_request(struct mddev *mddev, struct bio * bi) struct r5conf *conf = mddev->private; const int rw = bio_data_dir(bi); struct stripe_request_ctx *ctx; + struct bio *front_bio; sector_t logical_sector; enum stripe_result res; int s, stripe_cnt; + bool split = false; bool on_wq; if (unlikely(bi->bi_opf & REQ_PREFLUSH)) { @@ -6170,6 +6259,18 @@ static bool raid5_make_request(struct mddev *mddev, struct bio * bi) return true; } + front_bio = bi; + bi = mddev_bio_split_at_reshape_offset(mddev, bi, NULL, + &conf->bio_split); + if (!bi) { + if (rw == WRITE) + md_write_end(mddev); + return true; + } + if (bi != front_bio) + split = true; + front_bio = bi; + logical_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1); bi->bi_next = NULL; @@ -6191,15 +6292,6 @@ static bool raid5_make_request(struct mddev *mddev, struct bio * bi) pr_debug("raid456: %s, logical %llu to %llu\n", __func__, bi->bi_iter.bi_sector, ctx->last_sector); - /* Bail out if conflicts with reshape and REQ_NOWAIT is set */ - if ((bi->bi_opf & REQ_NOWAIT) && - get_reshape_loc(mddev, conf, logical_sector) == LOC_INSIDE_RESHAPE) { - bio_wouldblock_error(bi); - if (rw == WRITE) - md_write_end(mddev); - mempool_free(ctx, conf->ctx_pool); - return true; - } md_account_bio(mddev, &bi); /* @@ -6270,6 +6362,11 @@ static bool raid5_make_request(struct mddev *mddev, struct bio * bi) bio_endio(bi); wait_for_completion(&done); + front_bio->bi_status = BLK_STS_OK; + if (split) { + submit_bio_noacct(front_bio); + return true; + } return false; } @@ -6419,6 +6516,13 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *sk || test_bit(MD_RECOVERY_INTR, &mddev->recovery)); if (atomic_read(&conf->reshape_stripes) != 0) return 0; + if (md_bitmap_enabled(mddev, false) && + mddev->bitmap_ops->reshape_mark && + conf->reshape_safe != conf->reshape_progress) { + mddev->bitmap_ops->reshape_mark(mddev, conf->reshape_safe, + conf->reshape_progress); + mddev->bitmap_ops->unplug(mddev, true); + } mddev->reshape_position = conf->reshape_progress; mddev->curr_resync_completed = sector_nr; if (!mddev->reshape_backwards) @@ -6427,8 +6531,8 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *sk if (rdev->raid_disk >= 0 && !test_bit(Journal, &rdev->flags) && !test_bit(In_sync, &rdev->flags) && - rdev->recovery_offset < sector_nr) - rdev->recovery_offset = sector_nr; + READ_ONCE(rdev->recovery_offset) < sector_nr) + WRITE_ONCE(rdev->recovery_offset, sector_nr); conf->reshape_checkpoint = jiffies; set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); @@ -6528,6 +6632,13 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *sk || test_bit(MD_RECOVERY_INTR, &mddev->recovery)); if (atomic_read(&conf->reshape_stripes) != 0) goto ret; + if (md_bitmap_enabled(mddev, false) && + mddev->bitmap_ops->reshape_mark && + conf->reshape_safe != conf->reshape_progress) { + mddev->bitmap_ops->reshape_mark(mddev, conf->reshape_safe, + conf->reshape_progress); + mddev->bitmap_ops->unplug(mddev, true); + } mddev->reshape_position = conf->reshape_progress; mddev->curr_resync_completed = sector_nr; if (!mddev->reshape_backwards) @@ -6536,8 +6647,8 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *sk if (rdev->raid_disk >= 0 && !test_bit(Journal, &rdev->flags) && !test_bit(In_sync, &rdev->flags) && - rdev->recovery_offset < sector_nr) - rdev->recovery_offset = sector_nr; + READ_ONCE(rdev->recovery_offset) < sector_nr) + WRITE_ONCE(rdev->recovery_offset, sector_nr); conf->reshape_checkpoint = jiffies; set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); md_wakeup_thread(mddev->thread); @@ -6849,12 +6960,14 @@ static void raid5d(struct md_thread *thread) if ( !list_empty(&conf->bitmap_list)) { /* Now is a good time to flush some bitmap updates */ - conf->seq_flush++; + int seq = conf->seq_flush + 1; + + WRITE_ONCE(conf->seq_flush, seq); spin_unlock_irq(&conf->device_lock); if (md_bitmap_enabled(mddev, true)) mddev->bitmap_ops->unplug(mddev, true); spin_lock_irq(&conf->device_lock); - conf->seq_write = conf->seq_flush; + conf->seq_write = seq; activate_bit_delay(conf, conf->temp_inactive_list); } raid5_activate_delayed(conf); @@ -7039,6 +7152,7 @@ raid5_store_stripe_size(struct mddev *mddev, const char *page, size_t len) { struct r5conf *conf; unsigned long new; + unsigned int noio_flags = 0; int err; int size; @@ -7079,6 +7193,7 @@ raid5_store_stripe_size(struct mddev *mddev, const char *page, size_t len) goto out_unlock; } + noio_flags = memalloc_noio_save(); mutex_lock(&conf->cache_size_mutex); size = conf->max_nr_stripes; @@ -7095,6 +7210,7 @@ raid5_store_stripe_size(struct mddev *mddev, const char *page, size_t len) mutex_unlock(&conf->cache_size_mutex); out_unlock: + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); return err ?: len; } @@ -8056,9 +8172,9 @@ static int raid5_run(struct mddev *mddev) /* Hack because v0.91 doesn't store recovery_offset properly. */ if (mddev->major_version == 0 && mddev->minor_version > 90) - rdev->recovery_offset = reshape_offset; + WRITE_ONCE(rdev->recovery_offset, reshape_offset); - if (rdev->recovery_offset < reshape_offset) { + if (READ_ONCE(rdev->recovery_offset) < reshape_offset) { /* We need to check old and new layout */ if (!only_parity(rdev->raid_disk, conf->algorithm, @@ -8213,10 +8329,10 @@ static int raid5_spare_active(struct mddev *mddev) for (i = 0; i < conf->raid_disks; i++) { rdev = conf->disks[i].rdev; replacement = conf->disks[i].replacement; - if (replacement - && replacement->recovery_offset == MaxSector - && !test_bit(Faulty, &replacement->flags) - && !test_and_set_bit(In_sync, &replacement->flags)) { + if (replacement && + READ_ONCE(replacement->recovery_offset) == MaxSector && + !test_bit(Faulty, &replacement->flags) && + !test_and_set_bit(In_sync, &replacement->flags)) { /* Replacement has just become active. */ if (!rdev || !test_and_clear_bit(In_sync, &rdev->flags)) @@ -8231,10 +8347,10 @@ static int raid5_spare_active(struct mddev *mddev) rdev->sysfs_state); } sysfs_notify_dirent_safe(replacement->sysfs_state); - } else if (rdev - && rdev->recovery_offset == MaxSector - && !test_bit(Faulty, &rdev->flags) - && !test_and_set_bit(In_sync, &rdev->flags)) { + } else if (rdev && + READ_ONCE(rdev->recovery_offset) == MaxSector && + !test_bit(Faulty, &rdev->flags) && + !test_and_set_bit(In_sync, &rdev->flags)) { count++; sysfs_notify_dirent_safe(rdev->sysfs_state); } @@ -8479,6 +8595,8 @@ static int check_reshape(struct mddev *mddev) return 0; /* nothing to do */ if (has_failed(conf)) return -EINVAL; + if (!mddev->new_chunk_sectors) + return -EINVAL; if (mddev->delta_disks < 0 && mddev->reshape_position == MaxSector) { /* We might be able to shrink, but the devices must * be made bigger first. @@ -8495,6 +8613,9 @@ static int check_reshape(struct mddev *mddev) if (!check_stripe_cache(mddev)) return -ENOSPC; + if (mddev->bitmap_id == ID_LLBITMAP && + mddev->new_chunk_sectors < mddev->chunk_sectors) + return -EOPNOTSUPP; if (mddev->new_chunk_sectors > mddev->chunk_sectors || mddev->delta_disks > 0) if (resize_chunks(conf, @@ -8522,6 +8643,9 @@ static int raid5_start_reshape(struct mddev *mddev) if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) return -EBUSY; + if (!mddev->new_chunk_sectors) + return -EINVAL; + if (!check_stripe_cache(mddev)) return -ENOSPC; @@ -8557,6 +8681,12 @@ static int raid5_start_reshape(struct mddev *mddev) mdname(mddev)); return -EINVAL; } + if (md_bitmap_enabled(mddev, false) && + mddev->bitmap_id == ID_LLBITMAP) { + i = mddev->bitmap_ops->resize(mddev, mddev->dev_sectors, 0); + if (i) + return i; + } atomic_set(&conf->reshape_stripes, 0); spin_lock_irq(&conf->device_lock); @@ -8603,7 +8733,7 @@ static int raid5_start_reshape(struct mddev *mddev) >= conf->previous_raid_disks) set_bit(In_sync, &rdev->flags); else - rdev->recovery_offset = 0; + WRITE_ONCE(rdev->recovery_offset, 0); /* Failure here is OK */ sysfs_link_rdev(mddev, rdev); @@ -8641,10 +8771,19 @@ static int raid5_start_reshape(struct mddev *mddev) */ static void end_reshape(struct r5conf *conf) { + struct mddev *mddev = conf->mddev; if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) { struct md_rdev *rdev; + if (md_bitmap_enabled(mddev, false) && + mddev->bitmap_ops->reshape_mark && + conf->reshape_safe != conf->reshape_progress) { + mddev->bitmap_ops->reshape_mark(mddev, conf->reshape_safe, + conf->reshape_progress); + mddev->bitmap_ops->unplug(mddev, true); + } + spin_lock_irq(&conf->device_lock); conf->previous_raid_disks = conf->raid_disks; md_finish_reshape(conf->mddev); @@ -8655,7 +8794,7 @@ static void end_reshape(struct r5conf *conf) if (rdev->raid_disk >= 0 && !test_bit(Journal, &rdev->flags) && !test_bit(In_sync, &rdev->flags)) - rdev->recovery_offset = MaxSector; + WRITE_ONCE(rdev->recovery_offset, MaxSector); spin_unlock_irq(&conf->device_lock); wake_up(&conf->wait_for_reshape); @@ -8671,8 +8810,16 @@ static void raid5_finish_reshape(struct mddev *mddev) { struct r5conf *conf = mddev->private; struct md_rdev *rdev; + bool llbitmap = mddev->bitmap_id == ID_LLBITMAP && + md_bitmap_enabled(mddev, false); if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) { + if (llbitmap && mddev->bitmap_ops->reshape_finish) + mddev->bitmap_ops->reshape_finish(mddev); + if (llbitmap) { + mddev->resync_offset = 0; + mddev->resync_max_sectors = mddev->dev_sectors; + } if (mddev->delta_disks <= 0) { int d; @@ -8975,6 +9122,7 @@ static void *raid6_takeover(struct mddev *mddev) static int raid5_change_consistency_policy(struct mddev *mddev, const char *buf) { struct r5conf *conf; + unsigned int noio_flags; int err; err = mddev_suspend_and_lock(mddev); @@ -8986,6 +9134,7 @@ static int raid5_change_consistency_policy(struct mddev *mddev, const char *buf) return -ENODEV; } + noio_flags = memalloc_noio_save(); if (strncmp(buf, "ppl", 3) == 0) { /* ppl only works with RAID 5 */ if (!raid5_has_ppl(conf) && conf->level == 5) { @@ -9025,6 +9174,7 @@ static int raid5_change_consistency_policy(struct mddev *mddev, const char *buf) if (!err) md_update_sb(mddev, 1); + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); return err; @@ -9049,6 +9199,21 @@ static void raid5_prepare_suspend(struct mddev *mddev) wake_up(&conf->wait_for_reshape); } +static sector_t raid5_bitmap_sync_size(struct mddev *mddev, bool previous) +{ + return mddev->dev_sectors; +} + +static sector_t raid5_bitmap_array_sectors(struct mddev *mddev, bool previous) +{ + struct r5conf *conf = mddev->private; + + if (previous) + return raid5_size(mddev, mddev->dev_sectors, + conf->previous_raid_disks); + return raid5_size(mddev, mddev->dev_sectors, conf->raid_disks); +} + static struct md_personality raid6_personality = { .head = { @@ -9078,6 +9243,9 @@ static struct md_personality raid6_personality = .change_consistency_policy = raid5_change_consistency_policy, .prepare_suspend = raid5_prepare_suspend, .bitmap_sector = raid5_bitmap_sector, + .bitmap_sector_map = raid5_bitmap_sector_map, + .bitmap_sync_size = raid5_bitmap_sync_size, + .bitmap_array_sectors = raid5_bitmap_array_sectors, }; static struct md_personality raid5_personality = { @@ -9108,6 +9276,9 @@ static struct md_personality raid5_personality = .change_consistency_policy = raid5_change_consistency_policy, .prepare_suspend = raid5_prepare_suspend, .bitmap_sector = raid5_bitmap_sector, + .bitmap_sector_map = raid5_bitmap_sector_map, + .bitmap_sync_size = raid5_bitmap_sync_size, + .bitmap_array_sectors = raid5_bitmap_array_sectors, }; static struct md_personality raid4_personality = @@ -9139,6 +9310,9 @@ static struct md_personality raid4_personality = .change_consistency_policy = raid5_change_consistency_policy, .prepare_suspend = raid5_prepare_suspend, .bitmap_sector = raid5_bitmap_sector, + .bitmap_sector_map = raid5_bitmap_sector_map, + .bitmap_sync_size = raid5_bitmap_sync_size, + .bitmap_array_sectors = raid5_bitmap_array_sectors, }; static int __init raid5_init(void)