github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-05-25 15:41:52 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge branch 'xfs-6.15-zoned_devices' into XFS-for-linus-6.15-merge

Browse Source 
Merge Zoned allocator for XFS.

Signed-off-by: Carlos Maiolino <cem@kernel.org>
This commit is contained in:
Carlos Maiolino 2025-03-18 14:06:04 +01:00
commit 8e6415460f
91 changed files with 6394 additions and 1391 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Documentation/filesystems/iomap/design.rst
View File

@ -246,6 +246,10 @@ The fields are as follows:
* **IOMAP_F_PRIVATE**: Starting with this value, the upper bits can
be set by the filesystem for its own purposes.
* **IOMAP_F_ANON_WRITE**: Indicates that (write) I/O does not have a target
block assigned to it yet and the file system will do that in the bio
submission handler, splitting the I/O as needed.
These flags can be set by iomap itself during file operations.
The filesystem should supply an ``->iomap_end`` function if it needs
to observe these flags:
@ -352,6 +356,11 @@ operations:
``IOMAP_NOWAIT`` is often set on behalf of ``IOCB_NOWAIT`` or
``RWF_NOWAIT``.
* ``IOMAP_DONTCACHE`` is set when the caller wishes to perform a
buffered file I/O and would like the kernel to drop the pagecache
after the I/O completes, if it isn't already being used by another
thread.
If it is necessary to read existing file contents from a `different
<https://lore.kernel.org/all/20191008071527.29304-9-hch@lst.de/>`_
device or address range on a device, the filesystem should return that

13
Documentation/filesystems/iomap/operations.rst
View File

@ -131,6 +131,8 @@ These ``struct kiocb`` flags are significant for buffered I/O with iomap:
* ``IOCB_NOWAIT``: Turns on ``IOMAP_NOWAIT``.
* ``IOCB_DONTCACHE``: Turns on ``IOMAP_DONTCACHE``.
Internal per-Folio State
------------------------
@ -283,7 +285,7 @@ The ``ops`` structure must be specified and is as follows:
struct iomap_writeback_ops {
int (*map_blocks)(struct iomap_writepage_ctx *wpc, struct inode *inode,
loff_t offset, unsigned len);
int (*prepare_ioend)(struct iomap_ioend *ioend, int status);
int (*submit_ioend)(struct iomap_writepage_ctx *wpc, int status);
void (*discard_folio)(struct folio *folio, loff_t pos);
};
@ -306,13 +308,12 @@ The fields are as follows:
purpose.
This function must be supplied by the filesystem.
- ``prepare_ioend``: Enables filesystems to transform the writeback
ioend or perform any other preparatory work before the writeback I/O
is submitted.
- ``submit_ioend``: Allows the file systems to hook into writeback bio
submission.
This might include pre-write space accounting updates, or installing
a custom ``->bi_end_io`` function for internal purposes, such as
deferring the ioend completion to a workqueue to run metadata update
transactions from process context.
transactions from process context before submitting the bio.
This function is optional.
- ``discard_folio``: iomap calls this function after ``->map_blocks``
@ -341,7 +342,7 @@ This can happen in interrupt or process context, depending on the
storage device.
Filesystems that need to update internal bookkeeping (e.g. unwritten
extent conversions) should provide a ``->prepare_ioend`` function to
extent conversions) should provide a ``->submit_ioend`` function to
set ``struct iomap_end::bio::bi_end_io`` to its own function.
This function should call ``iomap_finish_ioends`` after finishing its
own work (e.g. unwritten extent conversion).

111
fs/dax.c
View File

@ -1258,7 +1258,7 @@ static vm_fault_t dax_pmd_load_hole(struct xa_state *xas, struct vm_fault *vmf,
}
#endif /* CONFIG_FS_DAX_PMD */
static s64 dax_unshare_iter(struct iomap_iter *iter)
static int dax_unshare_iter(struct iomap_iter *iter)
{
struct iomap *iomap = &iter->iomap;
const struct iomap *srcmap = iomap_iter_srcmap(iter);
@ -1266,11 +1266,11 @@ static s64 dax_unshare_iter(struct iomap_iter *iter)
u64 copy_len = iomap_length(iter);
u32 mod;
int id = 0;
s64 ret = 0;
s64 ret;
void *daddr = NULL, *saddr = NULL;
if (!iomap_want_unshare_iter(iter))
return iomap_length(iter);
return iomap_iter_advance_full(iter);
/*
* Extend the file range to be aligned to fsblock/pagesize, because
@ -1300,14 +1300,14 @@ static s64 dax_unshare_iter(struct iomap_iter *iter)
if (ret < 0)
goto out_unlock;
if (copy_mc_to_kernel(daddr, saddr, copy_len) == 0)
ret = iomap_length(iter);
else
if (copy_mc_to_kernel(daddr, saddr, copy_len) != 0)
ret = -EIO;
out_unlock:
dax_read_unlock(id);
return dax_mem2blk_err(ret);
if (ret < 0)
return dax_mem2blk_err(ret);
return iomap_iter_advance_full(iter);
}
int dax_file_unshare(struct inode *inode, loff_t pos, loff_t len,
@ -1326,7 +1326,7 @@ int dax_file_unshare(struct inode *inode, loff_t pos, loff_t len,
iter.len = min(len, size - pos);
while ((ret = iomap_iter(&iter, ops)) > 0)
iter.processed = dax_unshare_iter(&iter);
iter.status = dax_unshare_iter(&iter);
return ret;
}
EXPORT_SYMBOL_GPL(dax_file_unshare);
@ -1354,17 +1354,16 @@ static int dax_memzero(struct iomap_iter *iter, loff_t pos, size_t size)
return ret;
}
static s64 dax_zero_iter(struct iomap_iter *iter, bool *did_zero)
static int dax_zero_iter(struct iomap_iter *iter, bool *did_zero)
{
const struct iomap *iomap = &iter->iomap;
const struct iomap *srcmap = iomap_iter_srcmap(iter);
loff_t pos = iter->pos;
u64 length = iomap_length(iter);
s64 written = 0;
int ret;
/* already zeroed? we're done. */
if (srcmap->type == IOMAP_HOLE || srcmap->type == IOMAP_UNWRITTEN)
return length;
return iomap_iter_advance(iter, &length);
/*
* invalidate the pages whose sharing state is to be changed
@ -1372,33 +1371,35 @@ static s64 dax_zero_iter(struct iomap_iter *iter, bool *did_zero)
*/
if (iomap->flags & IOMAP_F_SHARED)
invalidate_inode_pages2_range(iter->inode->i_mapping,
pos >> PAGE_SHIFT,
(pos + length - 1) >> PAGE_SHIFT);
iter->pos >> PAGE_SHIFT,
(iter->pos + length - 1) >> PAGE_SHIFT);
do {
loff_t pos = iter->pos;
unsigned offset = offset_in_page(pos);
unsigned size = min_t(u64, PAGE_SIZE - offset, length);
pgoff_t pgoff = dax_iomap_pgoff(iomap, pos);
long rc;
int id;
length = min_t(u64, PAGE_SIZE - offset, length);
id = dax_read_lock();
if (IS_ALIGNED(pos, PAGE_SIZE) && size == PAGE_SIZE)
rc = dax_zero_page_range(iomap->dax_dev, pgoff, 1);
if (IS_ALIGNED(pos, PAGE_SIZE) && length == PAGE_SIZE)
ret = dax_zero_page_range(iomap->dax_dev, pgoff, 1);
else
rc = dax_memzero(iter, pos, size);
ret = dax_memzero(iter, pos, length);
dax_read_unlock(id);
if (rc < 0)
return rc;
pos += size;
length -= size;
written += size;
if (ret < 0)
return ret;
ret = iomap_iter_advance(iter, &length);
if (ret)
return ret;
} while (length > 0);
if (did_zero)
*did_zero = true;
return written;
return ret;
}
int dax_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero,
@ -1413,7 +1414,7 @@ int dax_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero,
int ret;
while ((ret = iomap_iter(&iter, ops)) > 0)
iter.processed = dax_zero_iter(&iter, did_zero);
iter.status = dax_zero_iter(&iter, did_zero);
return ret;
}
EXPORT_SYMBOL_GPL(dax_zero_range);
@ -1431,8 +1432,7 @@ int dax_truncate_page(struct inode *inode, loff_t pos, bool *did_zero,
}
EXPORT_SYMBOL_GPL(dax_truncate_page);
static loff_t dax_iomap_iter(const struct iomap_iter *iomi,
struct iov_iter *iter)
static int dax_iomap_iter(struct iomap_iter *iomi, struct iov_iter *iter)
{
const struct iomap *iomap = &iomi->iomap;
const struct iomap *srcmap = iomap_iter_srcmap(iomi);
@ -1451,8 +1451,10 @@ static loff_t dax_iomap_iter(const struct iomap_iter *iomi,
if (pos >= end)
return 0;
if (iomap->type == IOMAP_HOLE || iomap->type == IOMAP_UNWRITTEN)
return iov_iter_zero(min(length, end - pos), iter);
if (iomap->type == IOMAP_HOLE || iomap->type == IOMAP_UNWRITTEN) {
done = iov_iter_zero(min(length, end - pos), iter);
return iomap_iter_advance(iomi, &done);
}
}
/*
@ -1485,7 +1487,7 @@ static loff_t dax_iomap_iter(const struct iomap_iter *iomi,
}
id = dax_read_lock();
while (pos < end) {
while ((pos = iomi->pos) < end) {
unsigned offset = pos & (PAGE_SIZE - 1);
const size_t size = ALIGN(length + offset, PAGE_SIZE);
pgoff_t pgoff = dax_iomap_pgoff(iomap, pos);
@ -1535,18 +1537,16 @@ static loff_t dax_iomap_iter(const struct iomap_iter *iomi,
xfer = dax_copy_to_iter(dax_dev, pgoff, kaddr,
map_len, iter);
pos += xfer;
length -= xfer;
done += xfer;
if (xfer == 0)
length = xfer;
ret = iomap_iter_advance(iomi, &length);
if (!ret && xfer == 0)
ret = -EFAULT;
if (xfer < map_len)
break;
}
dax_read_unlock(id);
return done ? done : ret;
return ret;
}
/**
@ -1586,7 +1586,7 @@ dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter,
iomi.flags |= IOMAP_NOWAIT;
while ((ret = iomap_iter(&iomi, ops)) > 0)
iomi.processed = dax_iomap_iter(&iomi, iter);
iomi.status = dax_iomap_iter(&iomi, iter);
done = iomi.pos - iocb->ki_pos;
iocb->ki_pos = iomi.pos;
@ -1757,7 +1757,7 @@ static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, pfn_t *pfnp,
while ((error = iomap_iter(&iter, ops)) > 0) {
if (WARN_ON_ONCE(iomap_length(&iter) < PAGE_SIZE)) {
iter.processed = -EIO; /* fs corruption? */
iter.status = -EIO; /* fs corruption? */
continue;
}
@ -1769,8 +1769,10 @@ static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, pfn_t *pfnp,
ret |= VM_FAULT_MAJOR;
}
if (!(ret & VM_FAULT_ERROR))
iter.processed = PAGE_SIZE;
if (!(ret & VM_FAULT_ERROR)) {
u64 length = PAGE_SIZE;
iter.status = iomap_iter_advance(&iter, &length);
}
}
if (iomap_errp)
@ -1883,8 +1885,10 @@ static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp,
continue; /* actually breaks out of the loop */
ret = dax_fault_iter(vmf, &iter, pfnp, &xas, &entry, true);
if (ret != VM_FAULT_FALLBACK)
iter.processed = PMD_SIZE;
if (ret != VM_FAULT_FALLBACK) {
u64 length = PMD_SIZE;
iter.status = iomap_iter_advance(&iter, &length);
}
}
unlock_entry:
@ -1999,12 +2003,13 @@ vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf, unsigned int order,
}
EXPORT_SYMBOL_GPL(dax_finish_sync_fault);
static loff_t dax_range_compare_iter(struct iomap_iter *it_src,
static int dax_range_compare_iter(struct iomap_iter *it_src,
struct iomap_iter *it_dest, u64 len, bool *same)
{
const struct iomap *smap = &it_src->iomap;
const struct iomap *dmap = &it_dest->iomap;
loff_t pos1 = it_src->pos, pos2 = it_dest->pos;
u64 dest_len;
void *saddr, *daddr;
int id, ret;
@ -2012,7 +2017,7 @@ static loff_t dax_range_compare_iter(struct iomap_iter *it_src,
if (smap->type == IOMAP_HOLE && dmap->type == IOMAP_HOLE) {
*same = true;
return len;
goto advance;
}
if (smap->type == IOMAP_HOLE || dmap->type == IOMAP_HOLE) {
@ -2035,7 +2040,13 @@ static loff_t dax_range_compare_iter(struct iomap_iter *it_src,
if (!*same)
len = 0;
dax_read_unlock(id);
return len;
advance:
dest_len = len;
ret = iomap_iter_advance(it_src, &len);
if (!ret)
ret = iomap_iter_advance(it_dest, &dest_len);
return ret;
out_unlock:
dax_read_unlock(id);
@ -2058,15 +2069,15 @@ int dax_dedupe_file_range_compare(struct inode *src, loff_t srcoff,
.len = len,
.flags = IOMAP_DAX,
};
int ret, compared = 0;
int ret, status;
while ((ret = iomap_iter(&src_iter, ops)) > 0 &&
(ret = iomap_iter(&dst_iter, ops)) > 0) {
compared = dax_range_compare_iter(&src_iter, &dst_iter,
status = dax_range_compare_iter(&src_iter, &dst_iter,
min(src_iter.len, dst_iter.len), same);
if (compared < 0)
if (status < 0)
return ret;
src_iter.processed = dst_iter.processed = compared;
src_iter.status = dst_iter.status = status;
}
return ret;
}

3
fs/gfs2/bmap.c
View File

@ -1300,7 +1300,8 @@ static int gfs2_block_zero_range(struct inode *inode, loff_t from,
unsigned int length)
{
BUG_ON(current->journal_info);
return iomap_zero_range(inode, from, length, NULL, &gfs2_iomap_ops);
return iomap_zero_range(inode, from, length, NULL, &gfs2_iomap_ops,
NULL);
}
#define GFS2_JTRUNC_REVOKES 8192

1
fs/iomap/Makefile
View File

@ -12,6 +12,7 @@ iomap-y += trace.o \
iter.o
iomap-$(CONFIG_BLOCK) += buffered-io.o \
direct-io.o \
ioend.o \
fiemap.o \
seek.o
iomap-$(CONFIG_SWAP) += swapfile.o

345
fs/iomap/buffered-io.c
View File

@ -12,17 +12,15 @@
#include <linux/buffer_head.h>
#include <linux/dax.h>
#include <linux/writeback.h>
#include <linux/list_sort.h>
#include <linux/swap.h>
#include <linux/bio.h>
#include <linux/sched/signal.h>
#include <linux/migrate.h>
#include "internal.h"
#include "trace.h"
#include "../internal.h"
#define IOEND_BATCH_SIZE 4096
/*
* Structure allocated for each folio to track per-block uptodate, dirty state
* and I/O completions.
@ -40,8 +38,6 @@ struct iomap_folio_state {
unsigned long state[];
};
static struct bio_set iomap_ioend_bioset;
static inline bool ifs_is_fully_uptodate(struct folio *folio,
struct iomap_folio_state *ifs)
{
@ -366,15 +362,14 @@ static inline bool iomap_block_needs_zeroing(const struct iomap_iter *iter,
pos >= i_size_read(iter->inode);
}
static loff_t iomap_readpage_iter(const struct iomap_iter *iter,
struct iomap_readpage_ctx *ctx, loff_t offset)
static int iomap_readpage_iter(struct iomap_iter *iter,
struct iomap_readpage_ctx *ctx)
{
const struct iomap *iomap = &iter->iomap;
loff_t pos = iter->pos + offset;
loff_t length = iomap_length(iter) - offset;
loff_t pos = iter->pos;
loff_t length = iomap_length(iter);
struct folio *folio = ctx->cur_folio;
struct iomap_folio_state *ifs;
loff_t orig_pos = pos;
size_t poff, plen;
sector_t sector;
@ -438,25 +433,22 @@ static loff_t iomap_readpage_iter(const struct iomap_iter *iter,
* we can skip trailing ones as they will be handled in the next
* iteration.
*/
return pos - orig_pos + plen;
length = pos - iter->pos + plen;
return iomap_iter_advance(iter, &length);
}
static loff_t iomap_read_folio_iter(const struct iomap_iter *iter,
static int iomap_read_folio_iter(struct iomap_iter *iter,
struct iomap_readpage_ctx *ctx)
{
struct folio *folio = ctx->cur_folio;
size_t offset = offset_in_folio(folio, iter->pos);
loff_t length = min_t(loff_t, folio_size(folio) - offset,
iomap_length(iter));
loff_t done, ret;
int ret;
for (done = 0; done < length; done += ret) {
ret = iomap_readpage_iter(iter, ctx, done);
if (ret <= 0)
while (iomap_length(iter)) {
ret = iomap_readpage_iter(iter, ctx);
if (ret)
return ret;
}
return done;
return 0;
}
int iomap_read_folio(struct folio *folio, const struct iomap_ops *ops)
@ -474,7 +466,7 @@ int iomap_read_folio(struct folio *folio, const struct iomap_ops *ops)
trace_iomap_readpage(iter.inode, 1);
while ((ret = iomap_iter(&iter, ops)) > 0)
iter.processed = iomap_read_folio_iter(&iter, &ctx);
iter.status = iomap_read_folio_iter(&iter, &ctx);
if (ctx.bio) {
submit_bio(ctx.bio);
@ -493,15 +485,14 @@ int iomap_read_folio(struct folio *folio, const struct iomap_ops *ops)
}
EXPORT_SYMBOL_GPL(iomap_read_folio);
static loff_t iomap_readahead_iter(const struct iomap_iter *iter,
static int iomap_readahead_iter(struct iomap_iter *iter,
struct iomap_readpage_ctx *ctx)
{
loff_t length = iomap_length(iter);
loff_t done, ret;
int ret;
for (done = 0; done < length; done += ret) {
while (iomap_length(iter)) {
if (ctx->cur_folio &&
offset_in_folio(ctx->cur_folio, iter->pos + done) == 0) {
offset_in_folio(ctx->cur_folio, iter->pos) == 0) {
if (!ctx->cur_folio_in_bio)
folio_unlock(ctx->cur_folio);
ctx->cur_folio = NULL;
@ -510,12 +501,12 @@ static loff_t iomap_readahead_iter(const struct iomap_iter *iter,
ctx->cur_folio = readahead_folio(ctx->rac);
ctx->cur_folio_in_bio = false;
}
ret = iomap_readpage_iter(iter, ctx, done);
if (ret <= 0)
ret = iomap_readpage_iter(iter, ctx);
if (ret)
return ret;
}
return done;
return 0;
}
/**
@ -547,7 +538,7 @@ void iomap_readahead(struct readahead_control *rac, const struct iomap_ops *ops)
trace_iomap_readahead(rac->mapping->host, readahead_count(rac));
while (iomap_iter(&iter, ops) > 0)
iter.processed = iomap_readahead_iter(&iter, &ctx);
iter.status = iomap_readahead_iter(&iter, &ctx);
if (ctx.bio)
submit_bio(ctx.bio);
@ -603,6 +594,8 @@ struct folio *iomap_get_folio(struct iomap_iter *iter, loff_t pos, size_t len)
if (iter->flags & IOMAP_NOWAIT)
fgp |= FGP_NOWAIT;
if (iter->flags & IOMAP_DONTCACHE)
fgp |= FGP_DONTCACHE;
fgp |= fgf_set_order(len);
return __filemap_get_folio(iter->inode->i_mapping, pos >> PAGE_SHIFT,
@ -907,12 +900,10 @@ static bool iomap_write_end(struct iomap_iter *iter, loff_t pos, size_t len,
return __iomap_write_end(iter->inode, pos, len, copied, folio);
}
static loff_t iomap_write_iter(struct iomap_iter *iter, struct iov_iter *i)
static int iomap_write_iter(struct iomap_iter *iter, struct iov_iter *i)
{
loff_t length = iomap_length(iter);
loff_t pos = iter->pos;
ssize_t total_written = 0;
long status = 0;
int status = 0;
struct address_space *mapping = iter->inode->i_mapping;
size_t chunk = mapping_max_folio_size(mapping);
unsigned int bdp_flags = (iter->flags & IOMAP_NOWAIT) ? BDP_ASYNC : 0;
@ -923,7 +914,8 @@ static loff_t iomap_write_iter(struct iomap_iter *iter, struct iov_iter *i)
size_t offset; /* Offset into folio */
size_t bytes; /* Bytes to write to folio */
size_t copied; /* Bytes copied from user */
size_t written; /* Bytes have been written */
u64 written; /* Bytes have been written */
loff_t pos = iter->pos;
bytes = iov_iter_count(i);
retry:
@ -934,8 +926,8 @@ static loff_t iomap_write_iter(struct iomap_iter *iter, struct iov_iter *i)
if (unlikely(status))
break;
if (bytes > length)
bytes = length;
if (bytes > iomap_length(iter))
bytes = iomap_length(iter);
/*
* Bring in the user page that we'll copy from _first_.
@ -1006,17 +998,12 @@ static loff_t iomap_write_iter(struct iomap_iter *iter, struct iov_iter *i)
goto retry;
}
} else {
pos += written;
total_written += written;
length -= written;
iomap_iter_advance(iter, &written);
}
} while (iov_iter_count(i) && length);
} while (iov_iter_count(i) && iomap_length(iter));
if (status == -EAGAIN) {
iov_iter_revert(i, total_written);
return -EAGAIN;
}
return total_written ? total_written : status;
return total_written ? 0 : status;
}
ssize_t
@ -1034,9 +1021,11 @@ iomap_file_buffered_write(struct kiocb *iocb, struct iov_iter *i,
if (iocb->ki_flags & IOCB_NOWAIT)
iter.flags |= IOMAP_NOWAIT;
if (iocb->ki_flags & IOCB_DONTCACHE)
iter.flags |= IOMAP_DONTCACHE;
while ((ret = iomap_iter(&iter, ops)) > 0)
iter.processed = iomap_write_iter(&iter, i);
iter.status = iomap_write_iter(&iter, i);
if (unlikely(iter.pos == iocb->ki_pos))
return ret;
@ -1270,23 +1259,22 @@ void iomap_write_delalloc_release(struct inode *inode, loff_t start_byte,
}
EXPORT_SYMBOL_GPL(iomap_write_delalloc_release);
static loff_t iomap_unshare_iter(struct iomap_iter *iter)
static int iomap_unshare_iter(struct iomap_iter *iter)
{
struct iomap *iomap = &iter->iomap;
loff_t pos = iter->pos;
loff_t length = iomap_length(iter);
loff_t written = 0;
u64 bytes = iomap_length(iter);
int status;
if (!iomap_want_unshare_iter(iter))
return length;
return iomap_iter_advance(iter, &bytes);
do {
struct folio *folio;
int status;
size_t offset;
size_t bytes = min_t(u64, SIZE_MAX, length);
loff_t pos = iter->pos;
bool ret;
bytes = min_t(u64, SIZE_MAX, bytes);
status = iomap_write_begin(iter, pos, bytes, &folio);
if (unlikely(status))
return status;
@ -1304,14 +1292,14 @@ static loff_t iomap_unshare_iter(struct iomap_iter *iter)
cond_resched();
pos += bytes;
written += bytes;
length -= bytes;
balance_dirty_pages_ratelimited(iter->inode->i_mapping);
} while (length > 0);
return written;
status = iomap_iter_advance(iter, &bytes);
if (status)
break;
} while (bytes > 0);
return status;
}
int
@ -1331,7 +1319,7 @@ iomap_file_unshare(struct inode *inode, loff_t pos, loff_t len,
iter.len = min(len, size - pos);
while ((ret = iomap_iter(&iter, ops)) > 0)
iter.processed = iomap_unshare_iter(&iter);
iter.status = iomap_unshare_iter(&iter);
return ret;
}
EXPORT_SYMBOL_GPL(iomap_file_unshare);
@ -1350,19 +1338,18 @@ static inline int iomap_zero_iter_flush_and_stale(struct iomap_iter *i)
return filemap_write_and_wait_range(mapping, i->pos, end);
}
static loff_t iomap_zero_iter(struct iomap_iter *iter, bool *did_zero)
static int iomap_zero_iter(struct iomap_iter *iter, bool *did_zero)
{
loff_t pos = iter->pos;
loff_t length = iomap_length(iter);
loff_t written = 0;
u64 bytes = iomap_length(iter);
int status;
do {
struct folio *folio;
int status;
size_t offset;
size_t bytes = min_t(u64, SIZE_MAX, length);
loff_t pos = iter->pos;
bool ret;
bytes = min_t(u64, SIZE_MAX, bytes);
status = iomap_write_begin(iter, pos, bytes, &folio);
if (status)
return status;
@ -1383,25 +1370,26 @@ static loff_t iomap_zero_iter(struct iomap_iter *iter, bool *did_zero)
if (WARN_ON_ONCE(!ret))
return -EIO;
pos += bytes;
length -= bytes;
written += bytes;
} while (length > 0);
status = iomap_iter_advance(iter, &bytes);
if (status)
break;
} while (bytes > 0);
if (did_zero)
*did_zero = true;
return written;
return status;
}
int
iomap_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero,
const struct iomap_ops *ops)
const struct iomap_ops *ops, void *private)
{
struct iomap_iter iter = {
.inode = inode,
.pos = pos,
.len = len,
.flags = IOMAP_ZERO,
.private = private,
};
struct address_space *mapping = inode->i_mapping;
unsigned int blocksize = i_blocksize(inode);
@ -1424,7 +1412,7 @@ iomap_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero,
filemap_range_needs_writeback(mapping, pos, pos + plen - 1)) {
iter.len = plen;
while ((ret = iomap_iter(&iter, ops)) > 0)
iter.processed = iomap_zero_iter(&iter, did_zero);
iter.status = iomap_zero_iter(&iter, did_zero);
iter.len = len - (iter.pos - pos);
if (ret || !iter.len)
@ -1443,17 +1431,19 @@ iomap_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero,
if (srcmap->type == IOMAP_HOLE ||
srcmap->type == IOMAP_UNWRITTEN) {
loff_t proc = iomap_length(&iter);
s64 status;
if (range_dirty) {
range_dirty = false;
proc = iomap_zero_iter_flush_and_stale(&iter);
status = iomap_zero_iter_flush_and_stale(&iter);
} else {
status = iomap_iter_advance_full(&iter);
}
iter.processed = proc;
iter.status = status;
continue;
}
iter.processed = iomap_zero_iter(&iter, did_zero);
iter.status = iomap_zero_iter(&iter, did_zero);
}
return ret;
}
@ -1461,7 +1451,7 @@ EXPORT_SYMBOL_GPL(iomap_zero_range);
int
iomap_truncate_page(struct inode *inode, loff_t pos, bool *did_zero,
const struct iomap_ops *ops)
const struct iomap_ops *ops, void *private)
{
unsigned int blocksize = i_blocksize(inode);
unsigned int off = pos & (blocksize - 1);
@ -1469,11 +1459,12 @@ iomap_truncate_page(struct inode *inode, loff_t pos, bool *did_zero,
/* Block boundary? Nothing to do */
if (!off)
return 0;
return iomap_zero_range(inode, pos, blocksize - off, did_zero, ops);
return iomap_zero_range(inode, pos, blocksize - off, did_zero, ops,
private);
}
EXPORT_SYMBOL_GPL(iomap_truncate_page);
static loff_t iomap_folio_mkwrite_iter(struct iomap_iter *iter,
static int iomap_folio_mkwrite_iter(struct iomap_iter *iter,
struct folio *folio)
{
loff_t length = iomap_length(iter);
@ -1490,14 +1481,16 @@ static loff_t iomap_folio_mkwrite_iter(struct iomap_iter *iter,
folio_mark_dirty(folio);
}
return length;
return iomap_iter_advance(iter, &length);
}
vm_fault_t iomap_page_mkwrite(struct vm_fault *vmf, const struct iomap_ops *ops)
vm_fault_t iomap_page_mkwrite(struct vm_fault *vmf, const struct iomap_ops *ops,
void *private)
{
struct iomap_iter iter = {
.inode = file_inode(vmf->vma->vm_file),
.flags = IOMAP_WRITE | IOMAP_FAULT,
.private = private,
};
struct folio *folio = page_folio(vmf->page);
ssize_t ret;
@ -1509,7 +1502,7 @@ vm_fault_t iomap_page_mkwrite(struct vm_fault *vmf, const struct iomap_ops *ops)
iter.pos = folio_pos(folio);
iter.len = ret;
while ((ret = iomap_iter(&iter, ops)) > 0)
iter.processed = iomap_folio_mkwrite_iter(&iter, folio);
iter.status = iomap_folio_mkwrite_iter(&iter, folio);
if (ret < 0)
goto out_unlock;
@ -1538,16 +1531,15 @@ static void iomap_finish_folio_write(struct inode *inode, struct folio *folio,
* state, release holds on bios, and finally free up memory. Do not use the
* ioend after this.
*/
static u32
iomap_finish_ioend(struct iomap_ioend *ioend, int error)
u32 iomap_finish_ioend_buffered(struct iomap_ioend *ioend)
{
struct inode *inode = ioend->io_inode;
struct bio *bio = &ioend->io_bio;
struct folio_iter fi;
u32 folio_count = 0;
if (error) {
mapping_set_error(inode->i_mapping, error);
if (ioend->io_error) {
mapping_set_error(inode->i_mapping, ioend->io_error);
if (!bio_flagged(bio, BIO_QUIET)) {
pr_err_ratelimited(
"%s: writeback error on inode %lu, offset %lld, sector %llu",
@ -1566,116 +1558,16 @@ iomap_finish_ioend(struct iomap_ioend *ioend, int error)
return folio_count;
}
/*
* Ioend completion routine for merged bios. This can only be called from task
* contexts as merged ioends can be of unbound length. Hence we have to break up
* the writeback completions into manageable chunks to avoid long scheduler
* holdoffs. We aim to keep scheduler holdoffs down below 10ms so that we get
* good batch processing throughput without creating adverse scheduler latency
* conditions.
*/
void
iomap_finish_ioends(struct iomap_ioend *ioend, int error)
{
struct list_head tmp;
u32 completions;
might_sleep();
list_replace_init(&ioend->io_list, &tmp);
completions = iomap_finish_ioend(ioend, error);
while (!list_empty(&tmp)) {
if (completions > IOEND_BATCH_SIZE * 8) {
cond_resched();
completions = 0;
}
ioend = list_first_entry(&tmp, struct iomap_ioend, io_list);
list_del_init(&ioend->io_list);
completions += iomap_finish_ioend(ioend, error);
}
}
EXPORT_SYMBOL_GPL(iomap_finish_ioends);
/*
* We can merge two adjacent ioends if they have the same set of work to do.
*/
static bool
iomap_ioend_can_merge(struct iomap_ioend *ioend, struct iomap_ioend *next)
{
if (ioend->io_bio.bi_status != next->io_bio.bi_status)
return false;
if (next->io_flags & IOMAP_F_BOUNDARY)
return false;
if ((ioend->io_flags & IOMAP_F_SHARED) ^
(next->io_flags & IOMAP_F_SHARED))
return false;
if ((ioend->io_type == IOMAP_UNWRITTEN) ^
(next->io_type == IOMAP_UNWRITTEN))
return false;
if (ioend->io_offset + ioend->io_size != next->io_offset)
return false;
/*
* Do not merge physically discontiguous ioends. The filesystem
* completion functions will have to iterate the physical
* discontiguities even if we merge the ioends at a logical level, so
* we don't gain anything by merging physical discontiguities here.
*
* We cannot use bio->bi_iter.bi_sector here as it is modified during
* submission so does not point to the start sector of the bio at
* completion.
*/
if (ioend->io_sector + (ioend->io_size >> 9) != next->io_sector)
return false;
return true;
}
void
iomap_ioend_try_merge(struct iomap_ioend *ioend, struct list_head *more_ioends)
{
struct iomap_ioend *next;
INIT_LIST_HEAD(&ioend->io_list);
while ((next = list_first_entry_or_null(more_ioends, struct iomap_ioend,
io_list))) {
if (!iomap_ioend_can_merge(ioend, next))
break;
list_move_tail(&next->io_list, &ioend->io_list);
ioend->io_size += next->io_size;
}
}
EXPORT_SYMBOL_GPL(iomap_ioend_try_merge);
static int
iomap_ioend_compare(void *priv, const struct list_head *a,
const struct list_head *b)
{
struct iomap_ioend *ia = container_of(a, struct iomap_ioend, io_list);
struct iomap_ioend *ib = container_of(b, struct iomap_ioend, io_list);
if (ia->io_offset < ib->io_offset)
return -1;
if (ia->io_offset > ib->io_offset)
return 1;
return 0;
}
void
iomap_sort_ioends(struct list_head *ioend_list)
{
list_sort(NULL, ioend_list, iomap_ioend_compare);
}
EXPORT_SYMBOL_GPL(iomap_sort_ioends);
static void iomap_writepage_end_bio(struct bio *bio)
{
iomap_finish_ioend(iomap_ioend_from_bio(bio),
blk_status_to_errno(bio->bi_status));
struct iomap_ioend *ioend = iomap_ioend_from_bio(bio);
ioend->io_error = blk_status_to_errno(bio->bi_status);
iomap_finish_ioend_buffered(ioend);
}
/*
* Submit the final bio for an ioend.
* Submit an ioend.
*
* If @error is non-zero, it means that we have a situation where some part of
* the submission process has failed after we've marked pages for writeback.
@ -1694,14 +1586,18 @@ static int iomap_submit_ioend(struct iomap_writepage_ctx *wpc, int error)
* failure happened so that the file system end I/O handler gets called
* to clean up.
*/
if (wpc->ops->prepare_ioend)
error = wpc->ops->prepare_ioend(wpc->ioend, error);
if (wpc->ops->submit_ioend) {
error = wpc->ops->submit_ioend(wpc, error);
} else {
if (WARN_ON_ONCE(wpc->iomap.flags & IOMAP_F_ANON_WRITE))
error = -EIO;
if (!error)
submit_bio(&wpc->ioend->io_bio);
}
if (error) {
wpc->ioend->io_bio.bi_status = errno_to_blk_status(error);
bio_endio(&wpc->ioend->io_bio);
} else {
submit_bio(&wpc->ioend->io_bio);
}
wpc->ioend = NULL;
@ -1709,9 +1605,9 @@ static int iomap_submit_ioend(struct iomap_writepage_ctx *wpc, int error)
}
static struct iomap_ioend *iomap_alloc_ioend(struct iomap_writepage_ctx *wpc,
struct writeback_control *wbc, struct inode *inode, loff_t pos)
struct writeback_control *wbc, struct inode *inode, loff_t pos,
u16 ioend_flags)
{
struct iomap_ioend *ioend;
struct bio *bio;
bio = bio_alloc_bioset(wpc->iomap.bdev, BIO_MAX_VECS,
@ -1719,36 +1615,24 @@ static struct iomap_ioend *iomap_alloc_ioend(struct iomap_writepage_ctx *wpc,
GFP_NOFS, &iomap_ioend_bioset);
bio->bi_iter.bi_sector = iomap_sector(&wpc->iomap, pos);
bio->bi_end_io = iomap_writepage_end_bio;
wbc_init_bio(wbc, bio);
bio->bi_write_hint = inode->i_write_hint;
ioend = iomap_ioend_from_bio(bio);
INIT_LIST_HEAD(&ioend->io_list);
ioend->io_type = wpc->iomap.type;
ioend->io_flags = wpc->iomap.flags;
if (pos > wpc->iomap.offset)
wpc->iomap.flags &= ~IOMAP_F_BOUNDARY;
ioend->io_inode = inode;
ioend->io_size = 0;
ioend->io_offset = pos;
ioend->io_sector = bio->bi_iter.bi_sector;
wbc_init_bio(wbc, bio);
wpc->nr_folios = 0;
return ioend;
return iomap_init_ioend(inode, bio, pos, ioend_flags);
}
static bool iomap_can_add_to_ioend(struct iomap_writepage_ctx *wpc, loff_t pos)
static bool iomap_can_add_to_ioend(struct iomap_writepage_ctx *wpc, loff_t pos,
u16 ioend_flags)
{
if (wpc->iomap.offset == pos && (wpc->iomap.flags & IOMAP_F_BOUNDARY))
if (ioend_flags & IOMAP_IOEND_BOUNDARY)
return false;
if ((wpc->iomap.flags & IOMAP_F_SHARED) !=
(wpc->ioend->io_flags & IOMAP_F_SHARED))
return false;
if (wpc->iomap.type != wpc->ioend->io_type)
if ((ioend_flags & IOMAP_IOEND_NOMERGE_FLAGS) !=
(wpc->ioend->io_flags & IOMAP_IOEND_NOMERGE_FLAGS))
return false;
if (pos != wpc->ioend->io_offset + wpc->ioend->io_size)
return false;
if (iomap_sector(&wpc->iomap, pos) !=
if (!(wpc->iomap.flags & IOMAP_F_ANON_WRITE) &&
iomap_sector(&wpc->iomap, pos) !=
bio_end_sector(&wpc->ioend->io_bio))
return false;
/*
@ -1779,14 +1663,23 @@ static int iomap_add_to_ioend(struct iomap_writepage_ctx *wpc,
{
struct iomap_folio_state *ifs = folio->private;
size_t poff = offset_in_folio(folio, pos);
unsigned int ioend_flags = 0;
int error;
if (!wpc->ioend || !iomap_can_add_to_ioend(wpc, pos)) {
if (wpc->iomap.type == IOMAP_UNWRITTEN)
ioend_flags |= IOMAP_IOEND_UNWRITTEN;
if (wpc->iomap.flags & IOMAP_F_SHARED)
ioend_flags |= IOMAP_IOEND_SHARED;
if (pos == wpc->iomap.offset && (wpc->iomap.flags & IOMAP_F_BOUNDARY))
ioend_flags |= IOMAP_IOEND_BOUNDARY;
if (!wpc->ioend || !iomap_can_add_to_ioend(wpc, pos, ioend_flags)) {
new_ioend:
error = iomap_submit_ioend(wpc, 0);
if (error)
return error;
wpc->ioend = iomap_alloc_ioend(wpc, wbc, inode, pos);
wpc->ioend = iomap_alloc_ioend(wpc, wbc, inode, pos,
ioend_flags);
}
if (!bio_add_folio(&wpc->ioend->io_bio, folio, len, poff))
@ -2062,11 +1955,3 @@ iomap_writepages(struct address_space *mapping, struct writeback_control *wbc,
return iomap_submit_ioend(wpc, error);
}
EXPORT_SYMBOL_GPL(iomap_writepages);
static int __init iomap_buffered_init(void)
{
return bioset_init(&iomap_ioend_bioset, 4 * (PAGE_SIZE / SECTOR_SIZE),
offsetof(struct iomap_ioend, io_bio),
BIOSET_NEED_BVECS);
}
fs_initcall(iomap_buffered_init);

156
fs/iomap/direct-io.c
View File

@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2010 Red Hat, Inc.
* Copyright (c) 2016-2021 Christoph Hellwig.
* Copyright (c) 2016-2025 Christoph Hellwig.
*/
#include <linux/module.h>
#include <linux/compiler.h>
@ -12,6 +12,7 @@
#include <linux/backing-dev.h>
#include <linux/uio.h>
#include <linux/task_io_accounting_ops.h>
#include "internal.h"
#include "trace.h"
#include "../internal.h"
@ -20,6 +21,7 @@
* Private flags for iomap_dio, must not overlap with the public ones in
* iomap.h:
*/
#define IOMAP_DIO_NO_INVALIDATE (1U << 25)
#define IOMAP_DIO_CALLER_COMP (1U << 26)
#define IOMAP_DIO_INLINE_COMP (1U << 27)
#define IOMAP_DIO_WRITE_THROUGH (1U << 28)
@ -81,10 +83,12 @@ static void iomap_dio_submit_bio(const struct iomap_iter *iter,
WRITE_ONCE(iocb->private, bio);
}
if (dio->dops && dio->dops->submit_io)
if (dio->dops && dio->dops->submit_io) {
dio->dops->submit_io(iter, bio, pos);
else
} else {
WARN_ON_ONCE(iter->iomap.flags & IOMAP_F_ANON_WRITE);
submit_bio(bio);
}
}
ssize_t iomap_dio_complete(struct iomap_dio *dio)
@ -117,7 +121,8 @@ ssize_t iomap_dio_complete(struct iomap_dio *dio)
* ->end_io() when necessary, otherwise a racing buffer read would cache
* zeros from unwritten extents.
*/
if (!dio->error && dio->size && (dio->flags & IOMAP_DIO_WRITE))
if (!dio->error && dio->size && (dio->flags & IOMAP_DIO_WRITE) &&
!(dio->flags & IOMAP_DIO_NO_INVALIDATE))
kiocb_invalidate_post_direct_write(iocb, dio->size);
inode_dio_end(file_inode(iocb->ki_filp));
@ -163,43 +168,31 @@ static inline void iomap_dio_set_error(struct iomap_dio *dio, int ret)
cmpxchg(&dio->error, 0, ret);
}
void iomap_dio_bio_end_io(struct bio *bio)
/*
* Called when dio->ref reaches zero from an I/O completion.
*/
static void iomap_dio_done(struct iomap_dio *dio)
{
struct iomap_dio *dio = bio->bi_private;
bool should_dirty = (dio->flags & IOMAP_DIO_DIRTY);
struct kiocb *iocb = dio->iocb;
if (bio->bi_status)
iomap_dio_set_error(dio, blk_status_to_errno(bio->bi_status));
if (!atomic_dec_and_test(&dio->ref))
goto release_bio;
/*
* Synchronous dio, task itself will handle any completion work
* that needs after IO. All we need to do is wake the task.
*/
if (dio->wait_for_completion) {
/*
* Synchronous I/O, task itself will handle any completion work
* that needs after IO. All we need to do is wake the task.
*/
struct task_struct *waiter = dio->submit.waiter;
WRITE_ONCE(dio->submit.waiter, NULL);
blk_wake_io_task(waiter);
goto release_bio;
}
/*
* Flagged with IOMAP_DIO_INLINE_COMP, we can complete it inline
*/
if (dio->flags & IOMAP_DIO_INLINE_COMP) {
} else if (dio->flags & IOMAP_DIO_INLINE_COMP) {
WRITE_ONCE(iocb->private, NULL);
iomap_dio_complete_work(&dio->aio.work);
goto release_bio;
}
/*
* If this dio is flagged with IOMAP_DIO_CALLER_COMP, then schedule
* our completion that way to avoid an async punt to a workqueue.
*/
if (dio->flags & IOMAP_DIO_CALLER_COMP) {
} else if (dio->flags & IOMAP_DIO_CALLER_COMP) {
/*
* If this dio is flagged with IOMAP_DIO_CALLER_COMP, then
* schedule our completion that way to avoid an async punt to a
* workqueue.
*/
/* only polled IO cares about private cleared */
iocb->private = dio;
iocb->dio_complete = iomap_dio_deferred_complete;
@ -217,19 +210,31 @@ void iomap_dio_bio_end_io(struct bio *bio)
* issuer.
*/
iocb->ki_complete(iocb, 0);
goto release_bio;
}
} else {
struct inode *inode = file_inode(iocb->ki_filp);
/*
* Async DIO completion that requires filesystem level
* completion work gets punted to a work queue to complete as
* the operation may require more IO to be issued to finalise
* filesystem metadata changes or guarantee data integrity.
*/
INIT_WORK(&dio->aio.work, iomap_dio_complete_work);
queue_work(inode->i_sb->s_dio_done_wq, &dio->aio.work);
}
}
void iomap_dio_bio_end_io(struct bio *bio)
{
struct iomap_dio *dio = bio->bi_private;
bool should_dirty = (dio->flags & IOMAP_DIO_DIRTY);
if (bio->bi_status)
iomap_dio_set_error(dio, blk_status_to_errno(bio->bi_status));
if (atomic_dec_and_test(&dio->ref))
iomap_dio_done(dio);
/*
* Async DIO completion that requires filesystem level completion work
* gets punted to a work queue to complete as the operation may require
* more IO to be issued to finalise filesystem metadata changes or
* guarantee data integrity.
*/
INIT_WORK(&dio->aio.work, iomap_dio_complete_work);
queue_work(file_inode(iocb->ki_filp)->i_sb->s_dio_done_wq,
&dio->aio.work);
release_bio:
if (should_dirty) {
bio_check_pages_dirty(bio);
} else {
@ -239,6 +244,47 @@ void iomap_dio_bio_end_io(struct bio *bio)
}
EXPORT_SYMBOL_GPL(iomap_dio_bio_end_io);
u32 iomap_finish_ioend_direct(struct iomap_ioend *ioend)
{
struct iomap_dio *dio = ioend->io_bio.bi_private;
bool should_dirty = (dio->flags & IOMAP_DIO_DIRTY);
u32 vec_count = ioend->io_bio.bi_vcnt;
if (ioend->io_error)
iomap_dio_set_error(dio, ioend->io_error);
if (atomic_dec_and_test(&dio->ref)) {
/*
* Try to avoid another context switch for the completion given
* that we are already called from the ioend completion
* workqueue, but never invalidate pages from this thread to
* avoid deadlocks with buffered I/O completions. Tough luck if
* you hit the tiny race with someone dirtying the range now
* between this check and the actual completion.
*/
if (!dio->iocb->ki_filp->f_mapping->nrpages) {
dio->flags |= IOMAP_DIO_INLINE_COMP;
dio->flags |= IOMAP_DIO_NO_INVALIDATE;
}
dio->flags &= ~IOMAP_DIO_CALLER_COMP;
iomap_dio_done(dio);
}
if (should_dirty) {
bio_check_pages_dirty(&ioend->io_bio);
} else {
bio_release_pages(&ioend->io_bio, false);
bio_put(&ioend->io_bio);
}
/*
* Return the number of bvecs completed as even direct I/O completions
* do significant per-folio work and we'll still want to give up the
* CPU after a lot of completions.
*/
return vec_count;
}
static int iomap_dio_zero(const struct iomap_iter *iter, struct iomap_dio *dio,
loff_t pos, unsigned len)
{
@ -289,8 +335,7 @@ static inline blk_opf_t iomap_dio_bio_opflags(struct iomap_dio *dio,
return opflags;
}
static loff_t iomap_dio_bio_iter(const struct iomap_iter *iter,
struct iomap_dio *dio)
static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio)
{
const struct iomap *iomap = &iter->iomap;
struct inode *inode = iter->inode;
@ -303,7 +348,7 @@ static loff_t iomap_dio_bio_iter(const struct iomap_iter *iter,
bool need_zeroout = false;
bool use_fua = false;
int nr_pages, ret = 0;
size_t copied = 0;
u64 copied = 0;
size_t orig_count;
if (atomic && length != fs_block_size)
@ -465,30 +510,28 @@ static loff_t iomap_dio_bio_iter(const struct iomap_iter *iter,
/* Undo iter limitation to current extent */
iov_iter_reexpand(dio->submit.iter, orig_count - copied);
if (copied)
return copied;
return iomap_iter_advance(iter, &copied);
return ret;
}
static loff_t iomap_dio_hole_iter(const struct iomap_iter *iter,
struct iomap_dio *dio)
static int iomap_dio_hole_iter(struct iomap_iter *iter, struct iomap_dio *dio)
{
loff_t length = iov_iter_zero(iomap_length(iter), dio->submit.iter);
dio->size += length;
if (!length)
return -EFAULT;
return length;
return iomap_iter_advance(iter, &length);
}
static loff_t iomap_dio_inline_iter(const struct iomap_iter *iomi,
struct iomap_dio *dio)
static int iomap_dio_inline_iter(struct iomap_iter *iomi, struct iomap_dio *dio)
{
const struct iomap *iomap = &iomi->iomap;
struct iov_iter *iter = dio->submit.iter;
void *inline_data = iomap_inline_data(iomap, iomi->pos);
loff_t length = iomap_length(iomi);
loff_t pos = iomi->pos;
size_t copied;
u64 copied;
if (WARN_ON_ONCE(!iomap_inline_data_valid(iomap)))
return -EIO;
@ -510,11 +553,10 @@ static loff_t iomap_dio_inline_iter(const struct iomap_iter *iomi,
dio->size += copied;
if (!copied)
return -EFAULT;
return copied;
return iomap_iter_advance(iomi, &copied);
}
static loff_t iomap_dio_iter(const struct iomap_iter *iter,
struct iomap_dio *dio)
static int iomap_dio_iter(struct iomap_iter *iter, struct iomap_dio *dio)
{
switch (iter->iomap.type) {
case IOMAP_HOLE:
@ -698,7 +740,7 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
blk_start_plug(&plug);
while ((ret = iomap_iter(&iomi, ops)) > 0) {
iomi.processed = iomap_dio_iter(&iomi, dio);
iomi.status = iomap_dio_iter(&iomi, dio);
/*
* We can only poll for single bio I/Os.

21
fs/iomap/fiemap.c
View File

@ -39,24 +39,23 @@ static int iomap_to_fiemap(struct fiemap_extent_info *fi,
iomap->length, flags);
}
static loff_t iomap_fiemap_iter(const struct iomap_iter *iter,
static int iomap_fiemap_iter(struct iomap_iter *iter,
struct fiemap_extent_info *fi, struct iomap *prev)
{
int ret;
if (iter->iomap.type == IOMAP_HOLE)
return iomap_length(iter);
goto advance;
ret = iomap_to_fiemap(fi, prev, 0);
*prev = iter->iomap;
switch (ret) {
case 0: /* success */
return iomap_length(iter);
case 1: /* extent array full */
return 0;
default: /* error */
if (ret < 0)
return ret;
}
if (ret == 1) /* extent array full */
return 0;
advance:
return iomap_iter_advance_full(iter);
}
int iomap_fiemap(struct inode *inode, struct fiemap_extent_info *fi,
@ -78,7 +77,7 @@ int iomap_fiemap(struct inode *inode, struct fiemap_extent_info *fi,
return ret;
while ((ret = iomap_iter(&iter, ops)) > 0)
iter.processed = iomap_fiemap_iter(&iter, fi, &prev);
iter.status = iomap_fiemap_iter(&iter, fi, &prev);
if (prev.type != IOMAP_HOLE) {
ret = iomap_to_fiemap(fi, &prev, FIEMAP_EXTENT_LAST);
@ -114,7 +113,7 @@ iomap_bmap(struct address_space *mapping, sector_t bno,
while ((ret = iomap_iter(&iter, ops)) > 0) {
if (iter.iomap.type == IOMAP_MAPPED)
bno = iomap_sector(&iter.iomap, iter.pos) >> blkshift;
/* leave iter.processed unset to abort loop */
/* leave iter.status unset to abort loop */
}
if (ret)
return 0;

10
fs/iomap/internal.h Normal file
View File

@ -0,0 +1,10 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _IOMAP_INTERNAL_H
#define _IOMAP_INTERNAL_H 1
#define IOEND_BATCH_SIZE 4096
u32 iomap_finish_ioend_buffered(struct iomap_ioend *ioend);
u32 iomap_finish_ioend_direct(struct iomap_ioend *ioend);
#endif /* _IOMAP_INTERNAL_H */

216
fs/iomap/ioend.c Normal file
View File

@ -0,0 +1,216 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (c) 2024-2025 Christoph Hellwig.
*/
#include <linux/iomap.h>
#include <linux/list_sort.h>
#include "internal.h"
struct bio_set iomap_ioend_bioset;
EXPORT_SYMBOL_GPL(iomap_ioend_bioset);
struct iomap_ioend *iomap_init_ioend(struct inode *inode,
struct bio *bio, loff_t file_offset, u16 ioend_flags)
{
struct iomap_ioend *ioend = iomap_ioend_from_bio(bio);
atomic_set(&ioend->io_remaining, 1);
ioend->io_error = 0;
ioend->io_parent = NULL;
INIT_LIST_HEAD(&ioend->io_list);
ioend->io_flags = ioend_flags;
ioend->io_inode = inode;
ioend->io_offset = file_offset;
ioend->io_size = bio->bi_iter.bi_size;
ioend->io_sector = bio->bi_iter.bi_sector;
ioend->io_private = NULL;
return ioend;
}
EXPORT_SYMBOL_GPL(iomap_init_ioend);
static u32 iomap_finish_ioend(struct iomap_ioend *ioend, int error)
{
if (ioend->io_parent) {
struct bio *bio = &ioend->io_bio;
ioend = ioend->io_parent;
bio_put(bio);
}
if (error)
cmpxchg(&ioend->io_error, 0, error);
if (!atomic_dec_and_test(&ioend->io_remaining))
return 0;
if (ioend->io_flags & IOMAP_IOEND_DIRECT)
return iomap_finish_ioend_direct(ioend);
return iomap_finish_ioend_buffered(ioend);
}
/*
* Ioend completion routine for merged bios. This can only be called from task
* contexts as merged ioends can be of unbound length. Hence we have to break up
* the writeback completions into manageable chunks to avoid long scheduler
* holdoffs. We aim to keep scheduler holdoffs down below 10ms so that we get
* good batch processing throughput without creating adverse scheduler latency
* conditions.
*/
void iomap_finish_ioends(struct iomap_ioend *ioend, int error)
{
struct list_head tmp;
u32 completions;
might_sleep();
list_replace_init(&ioend->io_list, &tmp);
completions = iomap_finish_ioend(ioend, error);
while (!list_empty(&tmp)) {
if (completions > IOEND_BATCH_SIZE * 8) {
cond_resched();
completions = 0;
}
ioend = list_first_entry(&tmp, struct iomap_ioend, io_list);
list_del_init(&ioend->io_list);
completions += iomap_finish_ioend(ioend, error);
}
}
EXPORT_SYMBOL_GPL(iomap_finish_ioends);
/*
* We can merge two adjacent ioends if they have the same set of work to do.
*/
static bool iomap_ioend_can_merge(struct iomap_ioend *ioend,
struct iomap_ioend *next)
{
if (ioend->io_bio.bi_status != next->io_bio.bi_status)
return false;
if (next->io_flags & IOMAP_IOEND_BOUNDARY)
return false;
if ((ioend->io_flags & IOMAP_IOEND_NOMERGE_FLAGS) !=
(next->io_flags & IOMAP_IOEND_NOMERGE_FLAGS))
return false;
if (ioend->io_offset + ioend->io_size != next->io_offset)
return false;
/*
* Do not merge physically discontiguous ioends. The filesystem
* completion functions will have to iterate the physical
* discontiguities even if we merge the ioends at a logical level, so
* we don't gain anything by merging physical discontiguities here.
*
* We cannot use bio->bi_iter.bi_sector here as it is modified during
* submission so does not point to the start sector of the bio at
* completion.
*/
if (ioend->io_sector + (ioend->io_size >> SECTOR_SHIFT) !=
next->io_sector)
return false;
return true;
}
void iomap_ioend_try_merge(struct iomap_ioend *ioend,
struct list_head *more_ioends)
{
struct iomap_ioend *next;
INIT_LIST_HEAD(&ioend->io_list);
while ((next = list_first_entry_or_null(more_ioends, struct iomap_ioend,
io_list))) {
if (!iomap_ioend_can_merge(ioend, next))
break;
list_move_tail(&next->io_list, &ioend->io_list);
ioend->io_size += next->io_size;
}
}
EXPORT_SYMBOL_GPL(iomap_ioend_try_merge);
static int iomap_ioend_compare(void *priv, const struct list_head *a,
const struct list_head *b)
{
struct iomap_ioend *ia = container_of(a, struct iomap_ioend, io_list);
struct iomap_ioend *ib = container_of(b, struct iomap_ioend, io_list);
if (ia->io_offset < ib->io_offset)
return -1;
if (ia->io_offset > ib->io_offset)
return 1;
return 0;
}
void iomap_sort_ioends(struct list_head *ioend_list)
{
list_sort(NULL, ioend_list, iomap_ioend_compare);
}
EXPORT_SYMBOL_GPL(iomap_sort_ioends);
/*
* Split up to the first @max_len bytes from @ioend if the ioend covers more
* than @max_len bytes.
*
* If @is_append is set, the split will be based on the hardware limits for
* REQ_OP_ZONE_APPEND commands and can be less than @max_len if the hardware
* limits don't allow the entire @max_len length.
*
* The bio embedded into @ioend must be a REQ_OP_WRITE because the block layer
* does not allow splitting REQ_OP_ZONE_APPEND bios. The file systems has to
* switch the operation after this call, but before submitting the bio.
*/
struct iomap_ioend *iomap_split_ioend(struct iomap_ioend *ioend,
unsigned int max_len, bool is_append)
{
struct bio *bio = &ioend->io_bio;
struct iomap_ioend *split_ioend;
unsigned int nr_segs;
int sector_offset;
struct bio *split;
if (is_append) {
struct queue_limits *lim = bdev_limits(bio->bi_bdev);
max_len = min(max_len,
lim->max_zone_append_sectors << SECTOR_SHIFT);
sector_offset = bio_split_rw_at(bio, lim, &nr_segs, max_len);
if (unlikely(sector_offset < 0))
return ERR_PTR(sector_offset);
if (!sector_offset)
return NULL;
} else {
if (bio->bi_iter.bi_size <= max_len)
return NULL;
sector_offset = max_len >> SECTOR_SHIFT;
}
/* ensure the split ioend is still block size aligned */
sector_offset = ALIGN_DOWN(sector_offset << SECTOR_SHIFT,
i_blocksize(ioend->io_inode)) >> SECTOR_SHIFT;
split = bio_split(bio, sector_offset, GFP_NOFS, &iomap_ioend_bioset);
if (IS_ERR(split))
return ERR_CAST(split);
split->bi_private = bio->bi_private;
split->bi_end_io = bio->bi_end_io;
split_ioend = iomap_init_ioend(ioend->io_inode, split, ioend->io_offset,
ioend->io_flags);
split_ioend->io_parent = ioend;
atomic_inc(&ioend->io_remaining);
ioend->io_offset += split_ioend->io_size;
ioend->io_size -= split_ioend->io_size;
split_ioend->io_sector = ioend->io_sector;
if (!is_append)
ioend->io_sector += (split_ioend->io_size >> SECTOR_SHIFT);
return split_ioend;
}
EXPORT_SYMBOL_GPL(iomap_split_ioend);
static int __init iomap_ioend_init(void)
{
return bioset_init(&iomap_ioend_bioset, 4 * (PAGE_SIZE / SECTOR_SIZE),
offsetof(struct iomap_ioend, io_bio),
BIOSET_NEED_BVECS);
}
fs_initcall(iomap_ioend_init);

97
fs/iomap/iter.c
View File

@ -7,40 +7,25 @@
#include <linux/iomap.h>
#include "trace.h"
/*
* Advance to the next range we need to map.
*
* If the iomap is marked IOMAP_F_STALE, it means the existing map was not fully
* processed - it was aborted because the extent the iomap spanned may have been
* changed during the operation. In this case, the iteration behaviour is to
* remap the unprocessed range of the iter, and that means we may need to remap
* even when we've made no progress (i.e. iter->processed = 0). Hence the
* "finished iterating" case needs to distinguish between
* (processed = 0) meaning we are done and (processed = 0 && stale) meaning we
* need to remap the entire remaining range.
*/
static inline int iomap_iter_advance(struct iomap_iter *iter)
static inline void iomap_iter_reset_iomap(struct iomap_iter *iter)
{
bool stale = iter->iomap.flags & IOMAP_F_STALE;
int ret = 1;
/* handle the previous iteration (if any) */
if (iter->iomap.length) {
if (iter->processed < 0)
return iter->processed;
if (WARN_ON_ONCE(iter->processed > iomap_length(iter)))
return -EIO;
iter->pos += iter->processed;
iter->len -= iter->processed;
if (!iter->len || (!iter->processed && !stale))
ret = 0;
}
/* clear the per iteration state */
iter->processed = 0;
iter->status = 0;
memset(&iter->iomap, 0, sizeof(iter->iomap));
memset(&iter->srcmap, 0, sizeof(iter->srcmap));
return ret;
}
/*
* Advance the current iterator position and output the length remaining for the
* current mapping.
*/
int iomap_iter_advance(struct iomap_iter *iter, u64 *count)
{
if (WARN_ON_ONCE(*count > iomap_length(iter)))
return -EIO;
iter->pos += *count;
iter->len -= *count;
*count = iomap_length(iter);
return 0;
}
static inline void iomap_iter_done(struct iomap_iter *iter)
@ -50,6 +35,8 @@ static inline void iomap_iter_done(struct iomap_iter *iter)
WARN_ON_ONCE(iter->iomap.offset + iter->iomap.length <= iter->pos);
WARN_ON_ONCE(iter->iomap.flags & IOMAP_F_STALE);
iter->iter_start_pos = iter->pos;
trace_iomap_iter_dstmap(iter->inode, &iter->iomap);
if (iter->srcmap.type != IOMAP_HOLE)
trace_iomap_iter_srcmap(iter->inode, &iter->srcmap);
@ -67,26 +54,58 @@ static inline void iomap_iter_done(struct iomap_iter *iter)
* function must be called in a loop that continues as long it returns a
* positive value. If 0 or a negative value is returned, the caller must not
* return to the loop body. Within a loop body, there are two ways to break out
* of the loop body: leave @iter.processed unchanged, or set it to a negative
* of the loop body: leave @iter.status unchanged, or set it to a negative
* errno.
*/
int iomap_iter(struct iomap_iter *iter, const struct iomap_ops *ops)
{
bool stale = iter->iomap.flags & IOMAP_F_STALE;
ssize_t advanced;
u64 olen;
int ret;
if (iter->iomap.length && ops->iomap_end) {
ret = ops->iomap_end(iter->inode, iter->pos, iomap_length(iter),
iter->processed > 0 ? iter->processed : 0,
iter->flags, &iter->iomap);
if (ret < 0 && !iter->processed)
trace_iomap_iter(iter, ops, _RET_IP_);
if (!iter->iomap.length)
goto begin;
/*
* Calculate how far the iter was advanced and the original length bytes
* for ->iomap_end().
*/
advanced = iter->pos - iter->iter_start_pos;
olen = iter->len + advanced;
if (ops->iomap_end) {
ret = ops->iomap_end(iter->inode, iter->iter_start_pos,
iomap_length_trim(iter, iter->iter_start_pos,
olen),
advanced, iter->flags, &iter->iomap);
if (ret < 0 && !advanced)
return ret;
}
trace_iomap_iter(iter, ops, _RET_IP_);
ret = iomap_iter_advance(iter);
/* detect old return semantics where this would advance */
if (WARN_ON_ONCE(iter->status > 0))
iter->status = -EIO;
/*
* Use iter->len to determine whether to continue onto the next mapping.
* Explicitly terminate on error status or if the current iter has not
* advanced at all (i.e. no work was done for some reason) unless the
* mapping has been marked stale and needs to be reprocessed.
*/
if (iter->status < 0)
ret = iter->status;
else if (iter->len == 0 || (!advanced && !stale))
ret = 0;
else
ret = 1;
iomap_iter_reset_iomap(iter);
if (ret <= 0)
return ret;
begin:
ret = ops->iomap_begin(iter->inode, iter->pos, iter->len, iter->flags,
&iter->iomap, &iter->srcmap);
if (ret < 0)

16
fs/iomap/seek.c
View File

@ -10,7 +10,7 @@
#include <linux/pagemap.h>
#include <linux/pagevec.h>
static loff_t iomap_seek_hole_iter(const struct iomap_iter *iter,
static int iomap_seek_hole_iter(struct iomap_iter *iter,
loff_t *hole_pos)
{
loff_t length = iomap_length(iter);
@ -20,13 +20,13 @@ static loff_t iomap_seek_hole_iter(const struct iomap_iter *iter,
*hole_pos = mapping_seek_hole_data(iter->inode->i_mapping,
iter->pos, iter->pos + length, SEEK_HOLE);
if (*hole_pos == iter->pos + length)
return length;
return iomap_iter_advance(iter, &length);
return 0;
case IOMAP_HOLE:
*hole_pos = iter->pos;
return 0;
default:
return length;
return iomap_iter_advance(iter, &length);
}
}
@ -47,7 +47,7 @@ iomap_seek_hole(struct inode *inode, loff_t pos, const struct iomap_ops *ops)
iter.len = size - pos;
while ((ret = iomap_iter(&iter, ops)) > 0)
iter.processed = iomap_seek_hole_iter(&iter, &pos);
iter.status = iomap_seek_hole_iter(&iter, &pos);
if (ret < 0)
return ret;
if (iter.len) /* found hole before EOF */
@ -56,19 +56,19 @@ iomap_seek_hole(struct inode *inode, loff_t pos, const struct iomap_ops *ops)
}
EXPORT_SYMBOL_GPL(iomap_seek_hole);
static loff_t iomap_seek_data_iter(const struct iomap_iter *iter,
static int iomap_seek_data_iter(struct iomap_iter *iter,
loff_t *hole_pos)
{
loff_t length = iomap_length(iter);
switch (iter->iomap.type) {
case IOMAP_HOLE:
return length;
return iomap_iter_advance(iter, &length);
case IOMAP_UNWRITTEN:
*hole_pos = mapping_seek_hole_data(iter->inode->i_mapping,
iter->pos, iter->pos + length, SEEK_DATA);
if (*hole_pos < 0)
return length;
return iomap_iter_advance(iter, &length);
return 0;
default:
*hole_pos = iter->pos;
@ -93,7 +93,7 @@ iomap_seek_data(struct inode *inode, loff_t pos, const struct iomap_ops *ops)
iter.len = size - pos;
while ((ret = iomap_iter(&iter, ops)) > 0)
iter.processed = iomap_seek_data_iter(&iter, &pos);
iter.status = iomap_seek_data_iter(&iter, &pos);
if (ret < 0)
return ret;
if (iter.len) /* found data before EOF */

7
fs/iomap/swapfile.c
View File

@ -94,7 +94,7 @@ static int iomap_swapfile_fail(struct iomap_swapfile_info *isi, const char *str)
* swap only cares about contiguous page-aligned physical extents and makes no
* distinction between written and unwritten extents.
*/
static loff_t iomap_swapfile_iter(const struct iomap_iter *iter,
static int iomap_swapfile_iter(struct iomap_iter *iter,
struct iomap *iomap, struct iomap_swapfile_info *isi)
{
switch (iomap->type) {
@ -132,7 +132,8 @@ static loff_t iomap_swapfile_iter(const struct iomap_iter *iter,
return error;
memcpy(&isi->iomap, iomap, sizeof(isi->iomap));
}
return iomap_length(iter);
return iomap_iter_advance_full(iter);
}
/*
@ -166,7 +167,7 @@ int iomap_swapfile_activate(struct swap_info_struct *sis,
return ret;
while ((ret = iomap_iter(&iter, ops)) > 0)
iter.processed = iomap_swapfile_iter(&iter, &iter.iomap, &isi);
iter.status = iomap_swapfile_iter(&iter, &iter.iomap, &isi);
if (ret < 0)
return ret;

8
fs/iomap/trace.h
View File

@ -207,7 +207,7 @@ TRACE_EVENT(iomap_iter,
__field(u64, ino)
__field(loff_t, pos)
__field(u64, length)
__field(s64, processed)
__field(int, status)
__field(unsigned int, flags)
__field(const void *, ops)
__field(unsigned long, caller)
@ -217,17 +217,17 @@ TRACE_EVENT(iomap_iter,
__entry->ino = iter->inode->i_ino;
__entry->pos = iter->pos;
__entry->length = iomap_length(iter);
__entry->processed = iter->processed;
__entry->status = iter->status;
__entry->flags = iter->flags;
__entry->ops = ops;
__entry->caller = caller;
),
TP_printk("dev %d:%d ino 0x%llx pos 0x%llx length 0x%llx processed %lld flags %s (0x%x) ops %ps caller %pS",
TP_printk("dev %d:%d ino 0x%llx pos 0x%llx length 0x%llx status %d flags %s (0x%x) ops %ps caller %pS",
MAJOR(__entry->dev), MINOR(__entry->dev),
__entry->ino,
__entry->pos,
__entry->length,
__entry->processed,
__entry->status,
__print_flags(__entry->flags, "|", IOMAP_FLAGS_STRINGS),
__entry->flags,
__entry->ops,

7
fs/xfs/Makefile
View File

@ -64,6 +64,7 @@ xfs-y += $(addprefix libxfs/, \
xfs-$(CONFIG_XFS_RT) += $(addprefix libxfs/, \
xfs_rtbitmap.o \
xfs_rtgroup.o \
xfs_zones.o \
)
# highlevel code
@ -136,7 +137,11 @@ xfs-$(CONFIG_XFS_QUOTA) += xfs_dquot.o \
xfs_quotaops.o
# xfs_rtbitmap is shared with libxfs
xfs-$(CONFIG_XFS_RT) += xfs_rtalloc.o
xfs-$(CONFIG_XFS_RT) += xfs_rtalloc.o \
xfs_zone_alloc.o \
xfs_zone_gc.o \
xfs_zone_info.o \
xfs_zone_space_resv.o
xfs-$(CONFIG_XFS_POSIX_ACL) += xfs_acl.o
xfs-$(CONFIG_SYSCTL) += xfs_sysctl.o

316
fs/xfs/libxfs/xfs_bmap.c
View File

@ -34,13 +34,13 @@
#include "xfs_ag.h"
#include "xfs_ag_resv.h"
#include "xfs_refcount.h"
#include "xfs_icache.h"
#include "xfs_iomap.h"
#include "xfs_health.h"
#include "xfs_bmap_item.h"
#include "xfs_symlink_remote.h"
#include "xfs_inode_util.h"
#include "xfs_rtgroup.h"
#include "xfs_zone_alloc.h"
struct kmem_cache *xfs_bmap_intent_cache;
@ -171,18 +171,16 @@ xfs_bmbt_update(
* Compute the worst-case number of indirect blocks that will be used
* for ip's delayed extent of length "len".
*/
STATIC xfs_filblks_t
xfs_filblks_t
xfs_bmap_worst_indlen(
xfs_inode_t *ip, /* incore inode pointer */
xfs_filblks_t len) /* delayed extent length */
struct xfs_inode *ip, /* incore inode pointer */
xfs_filblks_t len) /* delayed extent length */
{
int level; /* btree level number */
int maxrecs; /* maximum record count at this level */
xfs_mount_t *mp; /* mount structure */
xfs_filblks_t rval; /* return value */
struct xfs_mount *mp = ip->i_mount;
int maxrecs = mp->m_bmap_dmxr[0];
int level;
xfs_filblks_t rval;
mp = ip->i_mount;
maxrecs = mp->m_bmap_dmxr[0];
for (level = 0, rval = 0;
level < XFS_BM_MAXLEVELS(mp, XFS_DATA_FORK);
level++) {
@ -2571,146 +2569,6 @@ xfs_bmap_add_extent_unwritten_real(
#undef PREV
}
/*
* Convert a hole to a delayed allocation.
*/
STATIC void
xfs_bmap_add_extent_hole_delay(
xfs_inode_t *ip, /* incore inode pointer */
int whichfork,
struct xfs_iext_cursor *icur,
xfs_bmbt_irec_t *new) /* new data to add to file extents */
{
struct xfs_ifork *ifp; /* inode fork pointer */
xfs_bmbt_irec_t left; /* left neighbor extent entry */
xfs_filblks_t newlen=0; /* new indirect size */
xfs_filblks_t oldlen=0; /* old indirect size */
xfs_bmbt_irec_t right; /* right neighbor extent entry */
uint32_t state = xfs_bmap_fork_to_state(whichfork);
xfs_filblks_t temp; /* temp for indirect calculations */
ifp = xfs_ifork_ptr(ip, whichfork);
ASSERT(isnullstartblock(new->br_startblock));
/*
* Check and set flags if this segment has a left neighbor
*/
if (xfs_iext_peek_prev_extent(ifp, icur, &left)) {
state |= BMAP_LEFT_VALID;
if (isnullstartblock(left.br_startblock))
state |= BMAP_LEFT_DELAY;
}
/*
* Check and set flags if the current (right) segment exists.
* If it doesn't exist, we're converting the hole at end-of-file.
*/
if (xfs_iext_get_extent(ifp, icur, &right)) {
state |= BMAP_RIGHT_VALID;
if (isnullstartblock(right.br_startblock))
state |= BMAP_RIGHT_DELAY;
}
/*
* Set contiguity flags on the left and right neighbors.
* Don't let extents get too large, even if the pieces are contiguous.
*/
if ((state & BMAP_LEFT_VALID) && (state & BMAP_LEFT_DELAY) &&
left.br_startoff + left.br_blockcount == new->br_startoff &&
left.br_blockcount + new->br_blockcount <= XFS_MAX_BMBT_EXTLEN)
state |= BMAP_LEFT_CONTIG;
if ((state & BMAP_RIGHT_VALID) && (state & BMAP_RIGHT_DELAY) &&
new->br_startoff + new->br_blockcount == right.br_startoff &&
new->br_blockcount + right.br_blockcount <= XFS_MAX_BMBT_EXTLEN &&
(!(state & BMAP_LEFT_CONTIG) ||
(left.br_blockcount + new->br_blockcount +
right.br_blockcount <= XFS_MAX_BMBT_EXTLEN)))
state |= BMAP_RIGHT_CONTIG;
/*
* Switch out based on the contiguity flags.
*/
switch (state & (BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG)) {
case BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG:
/*
* New allocation is contiguous with delayed allocations
* on the left and on the right.
* Merge all three into a single extent record.
*/
temp = left.br_blockcount + new->br_blockcount +
right.br_blockcount;
oldlen = startblockval(left.br_startblock) +
startblockval(new->br_startblock) +
startblockval(right.br_startblock);
newlen = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(ip, temp),
oldlen);
left.br_startblock = nullstartblock(newlen);
left.br_blockcount = temp;
xfs_iext_remove(ip, icur, state);
xfs_iext_prev(ifp, icur);
xfs_iext_update_extent(ip, state, icur, &left);
break;
case BMAP_LEFT_CONTIG:
/*
* New allocation is contiguous with a delayed allocation
* on the left.
* Merge the new allocation with the left neighbor.
*/
temp = left.br_blockcount + new->br_blockcount;
oldlen = startblockval(left.br_startblock) +
startblockval(new->br_startblock);
newlen = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(ip, temp),
oldlen);
left.br_blockcount = temp;
left.br_startblock = nullstartblock(newlen);
xfs_iext_prev(ifp, icur);
xfs_iext_update_extent(ip, state, icur, &left);
break;
case BMAP_RIGHT_CONTIG:
/*
* New allocation is contiguous with a delayed allocation
* on the right.
* Merge the new allocation with the right neighbor.
*/
temp = new->br_blockcount + right.br_blockcount;
oldlen = startblockval(new->br_startblock) +
startblockval(right.br_startblock);
newlen = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(ip, temp),
oldlen);
right.br_startoff = new->br_startoff;
right.br_startblock = nullstartblock(newlen);
right.br_blockcount = temp;
xfs_iext_update_extent(ip, state, icur, &right);
break;
case 0:
/*
* New allocation is not contiguous with another
* delayed allocation.
* Insert a new entry.
*/
oldlen = newlen = 0;
xfs_iext_insert(ip, icur, new, state);
break;
}
if (oldlen != newlen) {
ASSERT(oldlen > newlen);
xfs_add_fdblocks(ip->i_mount, oldlen - newlen);
/*
* Nothing to do for disk quota accounting here.
*/
xfs_mod_delalloc(ip, 0, (int64_t)newlen - oldlen);
}
}
/*
* Convert a hole to a real allocation.
*/
@ -4039,144 +3897,6 @@ xfs_bmapi_read(
return 0;
}
/*
* Add a delayed allocation extent to an inode. Blocks are reserved from the
* global pool and the extent inserted into the inode in-core extent tree.
*
* On entry, got refers to the first extent beyond the offset of the extent to
* allocate or eof is specified if no such extent exists. On return, got refers
* to the extent record that was inserted to the inode fork.
*
* Note that the allocated extent may have been merged with contiguous extents
* during insertion into the inode fork. Thus, got does not reflect the current
* state of the inode fork on return. If necessary, the caller can use lastx to
* look up the updated record in the inode fork.
*/
int
xfs_bmapi_reserve_delalloc(
struct xfs_inode *ip,
int whichfork,
xfs_fileoff_t off,
xfs_filblks_t len,
xfs_filblks_t prealloc,
struct xfs_bmbt_irec *got,
struct xfs_iext_cursor *icur,
int eof)
{
struct xfs_mount *mp = ip->i_mount;
struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork);
xfs_extlen_t alen;
xfs_extlen_t indlen;
uint64_t fdblocks;
int error;
xfs_fileoff_t aoff;
bool use_cowextszhint =
whichfork == XFS_COW_FORK && !prealloc;
retry:
/*
* Cap the alloc length. Keep track of prealloc so we know whether to
* tag the inode before we return.
*/
aoff = off;
alen = XFS_FILBLKS_MIN(len + prealloc, XFS_MAX_BMBT_EXTLEN);
if (!eof)
alen = XFS_FILBLKS_MIN(alen, got->br_startoff - aoff);
if (prealloc && alen >= len)
prealloc = alen - len;
/*
* If we're targetting the COW fork but aren't creating a speculative
* posteof preallocation, try to expand the reservation to align with
* the COW extent size hint if there's sufficient free space.
*
* Unlike the data fork, the CoW cancellation functions will free all
* the reservations at inactivation, so we don't require that every
* delalloc reservation have a dirty pagecache.
*/
if (use_cowextszhint) {
struct xfs_bmbt_irec prev;
xfs_extlen_t extsz = xfs_get_cowextsz_hint(ip);
if (!xfs_iext_peek_prev_extent(ifp, icur, &prev))
prev.br_startoff = NULLFILEOFF;
error = xfs_bmap_extsize_align(mp, got, &prev, extsz, 0, eof,
1, 0, &aoff, &alen);
ASSERT(!error);
}
/*
* Make a transaction-less quota reservation for delayed allocation
* blocks. This number gets adjusted later. We return if we haven't
* allocated blocks already inside this loop.
*/
error = xfs_quota_reserve_blkres(ip, alen);
if (error)
goto out;
/*
* Split changing sb for alen and indlen since they could be coming
* from different places.
*/
indlen = (xfs_extlen_t)xfs_bmap_worst_indlen(ip, alen);
ASSERT(indlen > 0);
fdblocks = indlen;
if (XFS_IS_REALTIME_INODE(ip)) {
error = xfs_dec_frextents(mp, xfs_blen_to_rtbxlen(mp, alen));
if (error)
goto out_unreserve_quota;
} else {
fdblocks += alen;
}
error = xfs_dec_fdblocks(mp, fdblocks, false);
if (error)
goto out_unreserve_frextents;
ip->i_delayed_blks += alen;
xfs_mod_delalloc(ip, alen, indlen);
got->br_startoff = aoff;
got->br_startblock = nullstartblock(indlen);
got->br_blockcount = alen;
got->br_state = XFS_EXT_NORM;
xfs_bmap_add_extent_hole_delay(ip, whichfork, icur, got);
/*
* Tag the inode if blocks were preallocated. Note that COW fork
* preallocation can occur at the start or end of the extent, even when
* prealloc == 0, so we must also check the aligned offset and length.
*/
if (whichfork == XFS_DATA_FORK && prealloc)
xfs_inode_set_eofblocks_tag(ip);
if (whichfork == XFS_COW_FORK && (prealloc || aoff < off || alen > len))
xfs_inode_set_cowblocks_tag(ip);
return 0;
out_unreserve_frextents:
if (XFS_IS_REALTIME_INODE(ip))
xfs_add_frextents(mp, xfs_blen_to_rtbxlen(mp, alen));
out_unreserve_quota:
if (XFS_IS_QUOTA_ON(mp))
xfs_quota_unreserve_blkres(ip, alen);
out:
if (error == -ENOSPC || error == -EDQUOT) {
trace_xfs_delalloc_enospc(ip, off, len);
if (prealloc || use_cowextszhint) {
/* retry without any preallocation */
use_cowextszhint = false;
prealloc = 0;
goto retry;
}
}
return error;
}
static int
xfs_bmapi_allocate(
struct xfs_bmalloca *bma)
@ -4948,7 +4668,8 @@ xfs_bmap_del_extent_delay(
int whichfork,
struct xfs_iext_cursor *icur,
struct xfs_bmbt_irec *got,
struct xfs_bmbt_irec *del)
struct xfs_bmbt_irec *del,
uint32_t bflags) /* bmapi flags */
{
struct xfs_mount *mp = ip->i_mount;
struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork);
@ -5068,10 +4789,18 @@ xfs_bmap_del_extent_delay(
da_diff = da_old - da_new;
fdblocks = da_diff;
if (isrt)
xfs_add_frextents(mp, xfs_blen_to_rtbxlen(mp, del->br_blockcount));
else
if (bflags & XFS_BMAPI_REMAP) {
;
} else if (isrt) {
xfs_rtbxlen_t rtxlen;
rtxlen = xfs_blen_to_rtbxlen(mp, del->br_blockcount);
if (xfs_is_zoned_inode(ip))
xfs_zoned_add_available(mp, rtxlen);
xfs_add_frextents(mp, rtxlen);
} else {
fdblocks += del->br_blockcount;
}
xfs_add_fdblocks(mp, fdblocks);
xfs_mod_delalloc(ip, -(int64_t)del->br_blockcount, -da_diff);
@ -5670,7 +5399,8 @@ __xfs_bunmapi(
delete:
if (wasdel) {
xfs_bmap_del_extent_delay(ip, whichfork, &icur, &got, &del);
xfs_bmap_del_extent_delay(ip, whichfork, &icur, &got,
&del, flags);
} else {
error = xfs_bmap_del_extent_real(ip, tp, &icur, cur,
&del, &tmp_logflags, whichfork,

7
fs/xfs/libxfs/xfs_bmap.h
View File

@ -204,7 +204,7 @@ int xfs_bunmapi(struct xfs_trans *tp, struct xfs_inode *ip,
xfs_extnum_t nexts, int *done);
void xfs_bmap_del_extent_delay(struct xfs_inode *ip, int whichfork,
struct xfs_iext_cursor *cur, struct xfs_bmbt_irec *got,
struct xfs_bmbt_irec *del);
struct xfs_bmbt_irec *del, uint32_t bflags);
void xfs_bmap_del_extent_cow(struct xfs_inode *ip,
struct xfs_iext_cursor *cur, struct xfs_bmbt_irec *got,
struct xfs_bmbt_irec *del);
@ -219,10 +219,6 @@ int xfs_bmap_insert_extents(struct xfs_trans *tp, struct xfs_inode *ip,
bool *done, xfs_fileoff_t stop_fsb);
int xfs_bmap_split_extent(struct xfs_trans *tp, struct xfs_inode *ip,
xfs_fileoff_t split_offset);
int xfs_bmapi_reserve_delalloc(struct xfs_inode *ip, int whichfork,
xfs_fileoff_t off, xfs_filblks_t len, xfs_filblks_t prealloc,
struct xfs_bmbt_irec *got, struct xfs_iext_cursor *cur,
int eof);
int xfs_bmapi_convert_delalloc(struct xfs_inode *ip, int whichfork,
xfs_off_t offset, struct iomap *iomap, unsigned int *seq);
int xfs_bmap_add_extent_unwritten_real(struct xfs_trans *tp,
@ -233,6 +229,7 @@ xfs_extlen_t xfs_bmapi_minleft(struct xfs_trans *tp, struct xfs_inode *ip,
int fork);
int xfs_bmap_btalloc_low_space(struct xfs_bmalloca *ap,
struct xfs_alloc_arg *args);
xfs_filblks_t xfs_bmap_worst_indlen(struct xfs_inode *ip, xfs_filblks_t len);
enum xfs_bmap_intent_type {
XFS_BMAP_MAP = 1,

20
fs/xfs/libxfs/xfs_format.h
View File

@ -178,9 +178,10 @@ typedef struct xfs_sb {
xfs_rgnumber_t sb_rgcount; /* number of realtime groups */
xfs_rtxlen_t sb_rgextents; /* size of a realtime group in rtx */
uint8_t sb_rgblklog; /* rt group number shift */
uint8_t sb_pad[7]; /* zeroes */
xfs_rfsblock_t sb_rtstart; /* start of internal RT section (FSB) */
xfs_filblks_t sb_rtreserved; /* reserved (zoned) RT blocks */
/* must be padded to 64 bit alignment */
} xfs_sb_t;
@ -270,9 +271,10 @@ struct xfs_dsb {
__be64 sb_metadirino; /* metadata directory tree root */
__be32 sb_rgcount; /* # of realtime groups */
__be32 sb_rgextents; /* size of rtgroup in rtx */
__u8 sb_rgblklog; /* rt group number shift */
__u8 sb_pad[7]; /* zeroes */
__be64 sb_rtstart; /* start of internal RT section (FSB) */
__be64 sb_rtreserved; /* reserved (zoned) RT blocks */
/*
* The size of this structure must be padded to 64 bit alignment.
@ -395,6 +397,9 @@ xfs_sb_has_ro_compat_feature(
#define XFS_SB_FEAT_INCOMPAT_EXCHRANGE (1 << 6) /* exchangerange supported */
#define XFS_SB_FEAT_INCOMPAT_PARENT (1 << 7) /* parent pointers */
#define XFS_SB_FEAT_INCOMPAT_METADIR (1 << 8) /* metadata dir tree */
#define XFS_SB_FEAT_INCOMPAT_ZONED (1 << 9) /* zoned RT allocator */
#define XFS_SB_FEAT_INCOMPAT_ZONE_GAPS (1 << 10) /* RTGs have LBA gaps */
#define XFS_SB_FEAT_INCOMPAT_ALL \
(XFS_SB_FEAT_INCOMPAT_FTYPE | \
XFS_SB_FEAT_INCOMPAT_SPINODES | \
@ -404,7 +409,9 @@ xfs_sb_has_ro_compat_feature(
XFS_SB_FEAT_INCOMPAT_NREXT64 | \
XFS_SB_FEAT_INCOMPAT_EXCHRANGE | \
XFS_SB_FEAT_INCOMPAT_PARENT | \
XFS_SB_FEAT_INCOMPAT_METADIR)
XFS_SB_FEAT_INCOMPAT_METADIR | \
XFS_SB_FEAT_INCOMPAT_ZONED | \
XFS_SB_FEAT_INCOMPAT_ZONE_GAPS)
#define XFS_SB_FEAT_INCOMPAT_UNKNOWN ~XFS_SB_FEAT_INCOMPAT_ALL
static inline bool
@ -952,7 +959,12 @@ struct xfs_dinode {
__be64 di_changecount; /* number of attribute changes */
__be64 di_lsn; /* flush sequence */
__be64 di_flags2; /* more random flags */
__be32 di_cowextsize; /* basic cow extent size for file */
union {
/* basic cow extent size for (regular) file */
__be32 di_cowextsize;
/* used blocks in RTG for (zoned) rtrmap inode */
__be32 di_used_blocks;
};
__u8 di_pad2[12]; /* more padding for future expansion */
/* fields only written to during inode creation */

14
fs/xfs/libxfs/xfs_fs.h
View File

@ -189,7 +189,9 @@ struct xfs_fsop_geom {
uint32_t checked; /* o: checked fs & rt metadata */
__u32 rgextents; /* rt extents in a realtime group */
__u32 rgcount; /* number of realtime groups */
__u64 reserved[16]; /* reserved space */
__u64 rtstart; /* start of internal rt section */
__u64 rtreserved; /* RT (zoned) reserved blocks */
__u64 reserved[14]; /* reserved space */
};
#define XFS_FSOP_GEOM_SICK_COUNTERS (1 << 0) /* summary counters */
@ -247,6 +249,7 @@ typedef struct xfs_fsop_resblks {
#define XFS_FSOP_GEOM_FLAGS_EXCHANGE_RANGE (1 << 24) /* exchange range */
#define XFS_FSOP_GEOM_FLAGS_PARENT (1 << 25) /* linux parent pointers */
#define XFS_FSOP_GEOM_FLAGS_METADIR (1 << 26) /* metadata directories */
#define XFS_FSOP_GEOM_FLAGS_ZONED (1 << 27) /* zoned rt device */
/*
* Minimum and maximum sizes need for growth checks.
@ -1079,6 +1082,15 @@ struct xfs_rtgroup_geometry {
#define XFS_IOC_COMMIT_RANGE _IOW ('X', 131, struct xfs_commit_range)
/* XFS_IOC_GETFSUUID ---------- deprecated 140 */
/*
* Devices supported by a single XFS file system. Reported in fsmaps fmr_device
* when using internal RT devices.
*/
enum xfs_device {
XFS_DEV_DATA = 1,
XFS_DEV_LOG = 2,
XFS_DEV_RT = 3,
};
#ifndef HAVE_BBMACROS
/*

31
fs/xfs/libxfs/xfs_group.h
View File

@ -19,10 +19,23 @@ struct xfs_group {
#ifdef __KERNEL__
/* -- kernel only structures below this line -- */
/*
* Track freed but not yet committed extents.
*/
struct xfs_extent_busy_tree *xg_busy_extents;
union {
/*
* For perags and non-zoned RT groups:
* Track freed but not yet committed extents.
*/
struct xfs_extent_busy_tree *xg_busy_extents;
/*
* For zoned RT groups:
* List of groups that need a zone reset.
*
* The zonegc code forces a log flush of the rtrmap inode before
* resetting the write pointer, so there is no need for
* individual busy extent tracking.
*/
struct xfs_group *xg_next_reset;
};
/*
* Bitsets of per-ag metadata that have been checked and/or are sick.
@ -107,9 +120,15 @@ xfs_gbno_to_daddr(
xfs_agblock_t gbno)
{
struct xfs_mount *mp = xg->xg_mount;
uint32_t blocks = mp->m_groups[xg->xg_type].blocks;
struct xfs_groups *g = &mp->m_groups[xg->xg_type];
xfs_fsblock_t fsbno;
return XFS_FSB_TO_BB(mp, (xfs_fsblock_t)xg->xg_gno * blocks + gbno);
if (g->has_daddr_gaps)
fsbno = xfs_gbno_to_fsb(xg, gbno);
else
fsbno = (xfs_fsblock_t)xg->xg_gno * g->blocks + gbno;
return XFS_FSB_TO_BB(mp, g->start_fsb + fsbno);
}
static inline uint32_t

2
fs/xfs/libxfs/xfs_ialloc.c
View File

@ -1927,7 +1927,7 @@ xfs_dialloc(
* that we can immediately allocate, but then we allow allocation on the
* second pass if we fail to find an AG with free inodes in it.
*/
if (percpu_counter_read_positive(&mp->m_fdblocks) <
if (xfs_estimate_freecounter(mp, XC_FREE_BLOCKS) <
mp->m_low_space[XFS_LOWSP_1_PCNT]) {
ok_alloc = false;
low_space = true;

21
fs/xfs/libxfs/xfs_inode_buf.c
View File

@ -252,7 +252,10 @@ xfs_inode_from_disk(
be64_to_cpu(from->di_changecount));
ip->i_crtime = xfs_inode_from_disk_ts(from, from->di_crtime);
ip->i_diflags2 = be64_to_cpu(from->di_flags2);
/* also covers the di_used_blocks union arm: */
ip->i_cowextsize = be32_to_cpu(from->di_cowextsize);
BUILD_BUG_ON(sizeof(from->di_cowextsize) !=
sizeof(from->di_used_blocks));
}
error = xfs_iformat_data_fork(ip, from);
@ -349,6 +352,7 @@ xfs_inode_to_disk(
to->di_changecount = cpu_to_be64(inode_peek_iversion(inode));
to->di_crtime = xfs_inode_to_disk_ts(ip, ip->i_crtime);
to->di_flags2 = cpu_to_be64(ip->i_diflags2);
/* also covers the di_used_blocks union arm: */
to->di_cowextsize = cpu_to_be32(ip->i_cowextsize);
to->di_ino = cpu_to_be64(ip->i_ino);
to->di_lsn = cpu_to_be64(lsn);
@ -752,11 +756,18 @@ xfs_dinode_verify(
!xfs_has_rtreflink(mp))
return __this_address;
/* COW extent size hint validation */
fa = xfs_inode_validate_cowextsize(mp, be32_to_cpu(dip->di_cowextsize),
mode, flags, flags2);
if (fa)
return fa;
if (xfs_has_zoned(mp) &&
dip->di_metatype == cpu_to_be16(XFS_METAFILE_RTRMAP)) {
if (be32_to_cpu(dip->di_used_blocks) > mp->m_sb.sb_rgextents)
return __this_address;
} else {
/* COW extent size hint validation */
fa = xfs_inode_validate_cowextsize(mp,
be32_to_cpu(dip->di_cowextsize),
mode, flags, flags2);
if (fa)
return fa;
}
/* bigtime iflag can only happen on bigtime filesystems */
if (xfs_dinode_has_bigtime(dip) &&

1
fs/xfs/libxfs/xfs_inode_util.c
View File

@ -322,6 +322,7 @@ xfs_inode_init(
if (xfs_has_v3inodes(mp)) {
inode_set_iversion(inode, 1);
/* also covers the di_used_blocks union arm: */
ip->i_cowextsize = 0;
times |= XFS_ICHGTIME_CREATE;
}

7
fs/xfs/libxfs/xfs_log_format.h
View File

@ -475,7 +475,12 @@ struct xfs_log_dinode {
xfs_lsn_t di_lsn;
uint64_t di_flags2; /* more random flags */
uint32_t di_cowextsize; /* basic cow extent size for file */
union {
/* basic cow extent size for (regular) file */
uint32_t di_cowextsize;
/* used blocks in RTG for (zoned) rtrmap inode */
uint32_t di_used_blocks;
};
uint8_t di_pad2[12]; /* more padding for future expansion */
/* fields only written to during inode creation */

175
fs/xfs/libxfs/xfs_metafile.c
View File

@ -21,6 +21,9 @@
#include "xfs_errortag.h"
#include "xfs_error.h"
#include "xfs_alloc.h"
#include "xfs_rtgroup.h"
#include "xfs_rtrmap_btree.h"
#include "xfs_rtrefcount_btree.h"
static const struct {
enum xfs_metafile_type mtype;
@ -74,12 +77,11 @@ xfs_metafile_clear_iflag(
}
/*
* Is the amount of space that could be allocated towards a given metadata
* file at or beneath a certain threshold?
* Is the metafile reservations at or beneath a certain threshold?
*/
static inline bool
xfs_metafile_resv_can_cover(
struct xfs_inode *ip,
struct xfs_mount *mp,
int64_t rhs)
{
/*
@ -88,43 +90,38 @@ xfs_metafile_resv_can_cover(
* global free block count. Take care of the first case to avoid
* touching the per-cpu counter.
*/
if (ip->i_delayed_blks >= rhs)
if (mp->m_metafile_resv_avail >= rhs)
return true;
/*
* There aren't enough blocks left in the inode's reservation, but it
* isn't critical unless there also isn't enough free space.
*/
return __percpu_counter_compare(&ip->i_mount->m_fdblocks,
rhs - ip->i_delayed_blks, 2048) >= 0;
return xfs_compare_freecounter(mp, XC_FREE_BLOCKS,
rhs - mp->m_metafile_resv_avail, 2048) >= 0;
}
/*
* Is this metadata file critically low on blocks? For now we'll define that
* as the number of blocks we can get our hands on being less than 10% of what
* we reserved or less than some arbitrary number (maximum btree height).
* Is the metafile reservation critically low on blocks? For now we'll define
* that as the number of blocks we can get our hands on being less than 10% of
* what we reserved or less than some arbitrary number (maximum btree height).
*/
bool
xfs_metafile_resv_critical(
struct xfs_inode *ip)
struct xfs_mount *mp)
{
uint64_t asked_low_water;
ASSERT(xfs_has_metadir(mp));
if (!ip)
return false;
trace_xfs_metafile_resv_critical(mp, 0);
ASSERT(xfs_is_metadir_inode(ip));
trace_xfs_metafile_resv_critical(ip, 0);
if (!xfs_metafile_resv_can_cover(ip, ip->i_mount->m_rtbtree_maxlevels))
if (!xfs_metafile_resv_can_cover(mp, mp->m_rtbtree_maxlevels))
return true;
asked_low_water = div_u64(ip->i_meta_resv_asked, 10);
if (!xfs_metafile_resv_can_cover(ip, asked_low_water))
if (!xfs_metafile_resv_can_cover(mp,
div_u64(mp->m_metafile_resv_target, 10)))
return true;
return XFS_TEST_ERROR(false, ip->i_mount,
XFS_ERRTAG_METAFILE_RESV_CRITICAL);
return XFS_TEST_ERROR(false, mp, XFS_ERRTAG_METAFILE_RESV_CRITICAL);
}
/* Allocate a block from the metadata file's reservation. */
@ -133,22 +130,24 @@ xfs_metafile_resv_alloc_space(
struct xfs_inode *ip,
struct xfs_alloc_arg *args)
{
struct xfs_mount *mp = ip->i_mount;
int64_t len = args->len;
ASSERT(xfs_is_metadir_inode(ip));
ASSERT(args->resv == XFS_AG_RESV_METAFILE);
trace_xfs_metafile_resv_alloc_space(ip, args->len);
trace_xfs_metafile_resv_alloc_space(mp, args->len);
/*
* Allocate the blocks from the metadata inode's block reservation
* and update the ondisk sb counter.
*/
if (ip->i_delayed_blks > 0) {
mutex_lock(&mp->m_metafile_resv_lock);
if (mp->m_metafile_resv_avail > 0) {
int64_t from_resv;
from_resv = min_t(int64_t, len, ip->i_delayed_blks);
ip->i_delayed_blks -= from_resv;
from_resv = min_t(int64_t, len, mp->m_metafile_resv_avail);
mp->m_metafile_resv_avail -= from_resv;
xfs_mod_delalloc(ip, 0, -from_resv);
xfs_trans_mod_sb(args->tp, XFS_TRANS_SB_RES_FDBLOCKS,
-from_resv);
@ -175,6 +174,9 @@ xfs_metafile_resv_alloc_space(
xfs_trans_mod_sb(args->tp, field, -len);
}
mp->m_metafile_resv_used += args->len;
mutex_unlock(&mp->m_metafile_resv_lock);
ip->i_nblocks += args->len;
xfs_trans_log_inode(args->tp, ip, XFS_ILOG_CORE);
}
@ -186,26 +188,33 @@ xfs_metafile_resv_free_space(
struct xfs_trans *tp,
xfs_filblks_t len)
{
struct xfs_mount *mp = ip->i_mount;
int64_t to_resv;
ASSERT(xfs_is_metadir_inode(ip));
trace_xfs_metafile_resv_free_space(ip, len);
trace_xfs_metafile_resv_free_space(mp, len);
ip->i_nblocks -= len;
xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
mutex_lock(&mp->m_metafile_resv_lock);
mp->m_metafile_resv_used -= len;
/*
* Add the freed blocks back into the inode's delalloc reservation
* until it reaches the maximum size. Update the ondisk fdblocks only.
*/
to_resv = ip->i_meta_resv_asked - (ip->i_nblocks + ip->i_delayed_blks);
to_resv = mp->m_metafile_resv_target -
(mp->m_metafile_resv_used + mp->m_metafile_resv_avail);
if (to_resv > 0) {
to_resv = min_t(int64_t, to_resv, len);
ip->i_delayed_blks += to_resv;
mp->m_metafile_resv_avail += to_resv;
xfs_mod_delalloc(ip, 0, to_resv);
xfs_trans_mod_sb(tp, XFS_TRANS_SB_RES_FDBLOCKS, to_resv);
len -= to_resv;
}
mutex_unlock(&mp->m_metafile_resv_lock);
/*
* Everything else goes back to the filesystem, so update the in-core
@ -215,61 +224,99 @@ xfs_metafile_resv_free_space(
xfs_trans_mod_sb(tp, XFS_TRANS_SB_FDBLOCKS, len);
}
/* Release a metadata file's space reservation. */
void
xfs_metafile_resv_free(
struct xfs_inode *ip)
static void
__xfs_metafile_resv_free(
struct xfs_mount *mp)
{
/* Non-btree metadata inodes don't need space reservations. */
if (!ip || !ip->i_meta_resv_asked)
return;
ASSERT(xfs_is_metadir_inode(ip));
trace_xfs_metafile_resv_free(ip, 0);
if (ip->i_delayed_blks) {
xfs_mod_delalloc(ip, 0, -ip->i_delayed_blks);
xfs_add_fdblocks(ip->i_mount, ip->i_delayed_blks);
ip->i_delayed_blks = 0;
if (mp->m_metafile_resv_avail) {
xfs_mod_sb_delalloc(mp, -(int64_t)mp->m_metafile_resv_avail);
xfs_add_fdblocks(mp, mp->m_metafile_resv_avail);
}
ip->i_meta_resv_asked = 0;
mp->m_metafile_resv_avail = 0;
mp->m_metafile_resv_used = 0;
mp->m_metafile_resv_target = 0;
}
/* Set up a metadata file's space reservation. */
/* Release unused metafile space reservation. */
void
xfs_metafile_resv_free(
struct xfs_mount *mp)
{
if (!xfs_has_metadir(mp))
return;
trace_xfs_metafile_resv_free(mp, 0);
mutex_lock(&mp->m_metafile_resv_lock);
__xfs_metafile_resv_free(mp);
mutex_unlock(&mp->m_metafile_resv_lock);
}
/* Set up a metafile space reservation. */
int
xfs_metafile_resv_init(
struct xfs_inode *ip,
xfs_filblks_t ask)
struct xfs_mount *mp)
{
struct xfs_rtgroup *rtg = NULL;
xfs_filblks_t used = 0, target = 0;
xfs_filblks_t hidden_space;
xfs_filblks_t used;
int error;
xfs_rfsblock_t dblocks_avail = mp->m_sb.sb_dblocks / 4;
int error = 0;
if (!ip || ip->i_meta_resv_asked > 0)
if (!xfs_has_metadir(mp))
return 0;
ASSERT(xfs_is_metadir_inode(ip));
/*
* Free any previous reservation to have a clean slate.
*/
mutex_lock(&mp->m_metafile_resv_lock);
__xfs_metafile_resv_free(mp);
/*
* Space taken by all other metadata btrees are accounted on-disk as
* Currently the only btree metafiles that require reservations are the
* rtrmap and the rtrefcount. Anything new will have to be added here
* as well.
*/
while ((rtg = xfs_rtgroup_next(mp, rtg))) {
if (xfs_has_rtrmapbt(mp)) {
used += rtg_rmap(rtg)->i_nblocks;
target += xfs_rtrmapbt_calc_reserves(mp);
}
if (xfs_has_rtreflink(mp)) {
used += rtg_refcount(rtg)->i_nblocks;
target += xfs_rtrefcountbt_calc_reserves(mp);
}
}
if (!target)
goto out_unlock;
/*
* Space taken by the per-AG metadata btrees are accounted on-disk as
* used space. We therefore only hide the space that is reserved but
* not used by the trees.
*/
used = ip->i_nblocks;
if (used > ask)
ask = used;
hidden_space = ask - used;
if (used > target)
target = used;
else if (target > dblocks_avail)
target = dblocks_avail;
hidden_space = target - used;
error = xfs_dec_fdblocks(ip->i_mount, hidden_space, true);
error = xfs_dec_fdblocks(mp, hidden_space, true);
if (error) {
trace_xfs_metafile_resv_init_error(ip, error, _RET_IP_);
return error;
trace_xfs_metafile_resv_init_error(mp, 0);
goto out_unlock;
}
xfs_mod_delalloc(ip, 0, hidden_space);
ip->i_delayed_blks = hidden_space;
ip->i_meta_resv_asked = ask;
xfs_mod_sb_delalloc(mp, hidden_space);
trace_xfs_metafile_resv_init(ip, ask);
return 0;
mp->m_metafile_resv_target = target;
mp->m_metafile_resv_used = used;
mp->m_metafile_resv_avail = hidden_space;
trace_xfs_metafile_resv_init(mp, target);
out_unlock:
mutex_unlock(&mp->m_metafile_resv_lock);
return error;
}

6
fs/xfs/libxfs/xfs_metafile.h
View File

@ -26,13 +26,13 @@ void xfs_metafile_clear_iflag(struct xfs_trans *tp, struct xfs_inode *ip);
/* Space reservations for metadata inodes. */
struct xfs_alloc_arg;
bool xfs_metafile_resv_critical(struct xfs_inode *ip);
bool xfs_metafile_resv_critical(struct xfs_mount *mp);
void xfs_metafile_resv_alloc_space(struct xfs_inode *ip,
struct xfs_alloc_arg *args);
void xfs_metafile_resv_free_space(struct xfs_inode *ip, struct xfs_trans *tp,
xfs_filblks_t len);
void xfs_metafile_resv_free(struct xfs_inode *ip);
int xfs_metafile_resv_init(struct xfs_inode *ip, xfs_filblks_t ask);
void xfs_metafile_resv_free(struct xfs_mount *mp);
int xfs_metafile_resv_init(struct xfs_mount *mp);
/* Code specific to kernel/userspace; must be provided externally. */

6
fs/xfs/libxfs/xfs_ondisk.h
View File

@ -233,8 +233,8 @@ xfs_check_ondisk_structs(void)
16299260424LL);
/* superblock field checks we got from xfs/122 */
XFS_CHECK_STRUCT_SIZE(struct xfs_dsb, 288);
XFS_CHECK_STRUCT_SIZE(struct xfs_sb, 288);
XFS_CHECK_STRUCT_SIZE(struct xfs_dsb, 304);
XFS_CHECK_STRUCT_SIZE(struct xfs_sb, 304);
XFS_CHECK_SB_OFFSET(sb_magicnum, 0);
XFS_CHECK_SB_OFFSET(sb_blocksize, 4);
XFS_CHECK_SB_OFFSET(sb_dblocks, 8);
@ -295,6 +295,8 @@ xfs_check_ondisk_structs(void)
XFS_CHECK_SB_OFFSET(sb_rgextents, 276);
XFS_CHECK_SB_OFFSET(sb_rgblklog, 280);
XFS_CHECK_SB_OFFSET(sb_pad, 281);
XFS_CHECK_SB_OFFSET(sb_rtstart, 288);
XFS_CHECK_SB_OFFSET(sb_rtreserved, 296);
}
#endif /* __XFS_ONDISK_H */

11
fs/xfs/libxfs/xfs_rtbitmap.c
View File

@ -1123,6 +1123,7 @@ xfs_rtfree_blocks(
xfs_extlen_t mod;
int error;
ASSERT(!xfs_has_zoned(mp));
ASSERT(rtlen <= XFS_MAX_BMBT_EXTLEN);
mod = xfs_blen_to_rtxoff(mp, rtlen);
@ -1174,6 +1175,9 @@ xfs_rtalloc_query_range(
end = min(end, rtg->rtg_extents - 1);
if (xfs_has_zoned(mp))
return -EINVAL;
/* Iterate the bitmap, looking for discrepancies. */
while (start <= end) {
struct xfs_rtalloc_rec rec;
@ -1268,6 +1272,8 @@ xfs_rtbitmap_blockcount_len(
struct xfs_mount *mp,
xfs_rtbxlen_t rtextents)
{
if (xfs_has_zoned(mp))
return 0;
return howmany_64(rtextents, xfs_rtbitmap_rtx_per_rbmblock(mp));
}
@ -1308,6 +1314,11 @@ xfs_rtsummary_blockcount(
xfs_rtbxlen_t rextents = xfs_rtbitmap_bitcount(mp);
unsigned long long rsumwords;
if (xfs_has_zoned(mp)) {
*rsumlevels = 0;
return 0;
}
*rsumlevels = xfs_compute_rextslog(rextents) + 1;
rsumwords = xfs_rtbitmap_blockcount_len(mp, rextents) * (*rsumlevels);
return howmany_64(rsumwords, mp->m_blockwsize);

39
fs/xfs/libxfs/xfs_rtgroup.c
View File

@ -194,15 +194,17 @@ xfs_rtgroup_lock(
ASSERT(!(rtglock_flags & XFS_RTGLOCK_BITMAP_SHARED) ||
!(rtglock_flags & XFS_RTGLOCK_BITMAP));
if (rtglock_flags & XFS_RTGLOCK_BITMAP) {
/*
* Lock both realtime free space metadata inodes for a freespace
* update.
*/
xfs_ilock(rtg_bitmap(rtg), XFS_ILOCK_EXCL);
xfs_ilock(rtg_summary(rtg), XFS_ILOCK_EXCL);
} else if (rtglock_flags & XFS_RTGLOCK_BITMAP_SHARED) {
xfs_ilock(rtg_bitmap(rtg), XFS_ILOCK_SHARED);
if (!xfs_has_zoned(rtg_mount(rtg))) {
if (rtglock_flags & XFS_RTGLOCK_BITMAP) {
/*
* Lock both realtime free space metadata inodes for a
* freespace update.
*/
xfs_ilock(rtg_bitmap(rtg), XFS_ILOCK_EXCL);
xfs_ilock(rtg_summary(rtg), XFS_ILOCK_EXCL);
} else if (rtglock_flags & XFS_RTGLOCK_BITMAP_SHARED) {
xfs_ilock(rtg_bitmap(rtg), XFS_ILOCK_SHARED);
}
}
if ((rtglock_flags & XFS_RTGLOCK_RMAP) && rtg_rmap(rtg))
@ -228,11 +230,13 @@ xfs_rtgroup_unlock(
if ((rtglock_flags & XFS_RTGLOCK_RMAP) && rtg_rmap(rtg))
xfs_iunlock(rtg_rmap(rtg), XFS_ILOCK_EXCL);
if (rtglock_flags & XFS_RTGLOCK_BITMAP) {
xfs_iunlock(rtg_summary(rtg), XFS_ILOCK_EXCL);
xfs_iunlock(rtg_bitmap(rtg), XFS_ILOCK_EXCL);
} else if (rtglock_flags & XFS_RTGLOCK_BITMAP_SHARED) {
xfs_iunlock(rtg_bitmap(rtg), XFS_ILOCK_SHARED);
if (!xfs_has_zoned(rtg_mount(rtg))) {
if (rtglock_flags & XFS_RTGLOCK_BITMAP) {
xfs_iunlock(rtg_summary(rtg), XFS_ILOCK_EXCL);
xfs_iunlock(rtg_bitmap(rtg), XFS_ILOCK_EXCL);
} else if (rtglock_flags & XFS_RTGLOCK_BITMAP_SHARED) {
xfs_iunlock(rtg_bitmap(rtg), XFS_ILOCK_SHARED);
}
}
}
@ -249,7 +253,8 @@ xfs_rtgroup_trans_join(
ASSERT(!(rtglock_flags & ~XFS_RTGLOCK_ALL_FLAGS));
ASSERT(!(rtglock_flags & XFS_RTGLOCK_BITMAP_SHARED));
if (rtglock_flags & XFS_RTGLOCK_BITMAP) {
if (!xfs_has_zoned(rtg_mount(rtg)) &&
(rtglock_flags & XFS_RTGLOCK_BITMAP)) {
xfs_trans_ijoin(tp, rtg_bitmap(rtg), XFS_ILOCK_EXCL);
xfs_trans_ijoin(tp, rtg_summary(rtg), XFS_ILOCK_EXCL);
}
@ -270,7 +275,7 @@ xfs_rtgroup_get_geometry(
/* Fill out form. */
memset(rgeo, 0, sizeof(*rgeo));
rgeo->rg_number = rtg_rgno(rtg);
rgeo->rg_length = rtg_group(rtg)->xg_block_count;
rgeo->rg_length = rtg_blocks(rtg);
xfs_rtgroup_geom_health(rtg, rgeo);
return 0;
}
@ -354,6 +359,7 @@ static const struct xfs_rtginode_ops xfs_rtginode_ops[XFS_RTGI_MAX] = {
.sick = XFS_SICK_RG_BITMAP,
.fmt_mask = (1U << XFS_DINODE_FMT_EXTENTS) |
(1U << XFS_DINODE_FMT_BTREE),
.enabled = xfs_has_nonzoned,
.create = xfs_rtbitmap_create,
},
[XFS_RTGI_SUMMARY] = {
@ -362,6 +368,7 @@ static const struct xfs_rtginode_ops xfs_rtginode_ops[XFS_RTGI_MAX] = {
.sick = XFS_SICK_RG_SUMMARY,
.fmt_mask = (1U << XFS_DINODE_FMT_EXTENTS) |
(1U << XFS_DINODE_FMT_BTREE),
.enabled = xfs_has_nonzoned,
.create = xfs_rtsummary_create,
},
[XFS_RTGI_RMAP] = {

50
fs/xfs/libxfs/xfs_rtgroup.h
View File

@ -37,15 +37,33 @@ struct xfs_rtgroup {
xfs_rtxnum_t rtg_extents;
/*
* Cache of rt summary level per bitmap block with the invariant that
* rtg_rsum_cache[bbno] > the maximum i for which rsum[i][bbno] != 0,
* or 0 if rsum[i][bbno] == 0 for all i.
*
* For bitmap based RT devices this points to a cache of rt summary
* level per bitmap block with the invariant that rtg_rsum_cache[bbno]
* > the maximum i for which rsum[i][bbno] != 0, or 0 if
* rsum[i][bbno] == 0 for all i.
* Reads and writes are serialized by the rsumip inode lock.
*
* For zoned RT devices this points to the open zone structure for
* a group that is open for writers, or is NULL.
*/
uint8_t *rtg_rsum_cache;
union {
uint8_t *rtg_rsum_cache;
struct xfs_open_zone *rtg_open_zone;
};
};
/*
* For zoned RT devices this is set on groups that have no written blocks
* and can be picked by the allocator for opening.
*/
#define XFS_RTG_FREE XA_MARK_0
/*
* For zoned RT devices this is set on groups that are fully written and that
* have unused blocks. Used by the garbage collection to pick targets.
*/
#define XFS_RTG_RECLAIMABLE XA_MARK_1
static inline struct xfs_rtgroup *to_rtg(struct xfs_group *xg)
{
return container_of(xg, struct xfs_rtgroup, rtg_group);
@ -66,6 +84,11 @@ static inline xfs_rgnumber_t rtg_rgno(const struct xfs_rtgroup *rtg)
return rtg->rtg_group.xg_gno;
}
static inline xfs_rgblock_t rtg_blocks(const struct xfs_rtgroup *rtg)
{
return rtg->rtg_group.xg_block_count;
}
static inline struct xfs_inode *rtg_bitmap(const struct xfs_rtgroup *rtg)
{
return rtg->rtg_inodes[XFS_RTGI_BITMAP];
@ -222,10 +245,14 @@ xfs_rtb_to_daddr(
xfs_rtblock_t rtbno)
{
struct xfs_groups *g = &mp->m_groups[XG_TYPE_RTG];
xfs_rgnumber_t rgno = xfs_rtb_to_rgno(mp, rtbno);
uint64_t start_bno = (xfs_rtblock_t)rgno * g->blocks;
return XFS_FSB_TO_BB(mp, start_bno + (rtbno & g->blkmask));
if (xfs_has_rtgroups(mp) && !g->has_daddr_gaps) {
xfs_rgnumber_t rgno = xfs_rtb_to_rgno(mp, rtbno);
rtbno = (xfs_rtblock_t)rgno * g->blocks + (rtbno & g->blkmask);
}
return XFS_FSB_TO_BB(mp, g->start_fsb + rtbno);
}
static inline xfs_rtblock_t
@ -233,10 +260,11 @@ xfs_daddr_to_rtb(
struct xfs_mount *mp,
xfs_daddr_t daddr)
{
xfs_rfsblock_t bno = XFS_BB_TO_FSBT(mp, daddr);
struct xfs_groups *g = &mp->m_groups[XG_TYPE_RTG];
xfs_rfsblock_t bno;
if (xfs_has_rtgroups(mp)) {
struct xfs_groups *g = &mp->m_groups[XG_TYPE_RTG];
bno = XFS_BB_TO_FSBT(mp, daddr) - g->start_fsb;
if (xfs_has_rtgroups(mp) && !g->has_daddr_gaps) {
xfs_rgnumber_t rgno;
uint32_t rgbno;

19
fs/xfs/libxfs/xfs_rtrmap_btree.c
View File

@ -1033,3 +1033,22 @@ xfs_rtrmapbt_init_rtsb(
xfs_btree_del_cursor(cur, error);
return error;
}
/*
* Return the highest rgbno currently tracked by the rmap for this rtg.
*/
xfs_rgblock_t
xfs_rtrmap_highest_rgbno(
struct xfs_rtgroup *rtg)
{
struct xfs_btree_block *block = rtg_rmap(rtg)->i_df.if_broot;
union xfs_btree_key key = {};
struct xfs_btree_cur *cur;
if (block->bb_numrecs == 0)
return NULLRGBLOCK;
cur = xfs_rtrmapbt_init_cursor(NULL, rtg);
xfs_btree_get_keys(cur, block, &key);
xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR);
return be32_to_cpu(key.__rmap_bigkey[1].rm_startblock);
}

2
fs/xfs/libxfs/xfs_rtrmap_btree.h
View File

@ -207,4 +207,6 @@ struct xfs_btree_cur *xfs_rtrmapbt_mem_cursor(struct xfs_rtgroup *rtg,
int xfs_rtrmapbt_mem_init(struct xfs_mount *mp, struct xfbtree *xfbtree,
struct xfs_buftarg *btp, xfs_rgnumber_t rgno);
xfs_rgblock_t xfs_rtrmap_highest_rgbno(struct xfs_rtgroup *rtg);
#endif /* __XFS_RTRMAP_BTREE_H__ */

81
fs/xfs/libxfs/xfs_sb.c
View File

@ -185,6 +185,8 @@ xfs_sb_version_to_features(
features |= XFS_FEAT_PARENT;
if (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_METADIR)
features |= XFS_FEAT_METADIR;
if (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_ZONED)
features |= XFS_FEAT_ZONED;
return features;
}
@ -266,6 +268,9 @@ static uint64_t
xfs_expected_rbmblocks(
struct xfs_sb *sbp)
{
if (xfs_sb_is_v5(sbp) &&
(sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_ZONED))
return 0;
return howmany_64(xfs_extents_per_rbm(sbp),
NBBY * xfs_rtbmblock_size(sbp));
}
@ -275,9 +280,15 @@ bool
xfs_validate_rt_geometry(
struct xfs_sb *sbp)
{
if (sbp->sb_rextsize * sbp->sb_blocksize > XFS_MAX_RTEXTSIZE ||
sbp->sb_rextsize * sbp->sb_blocksize < XFS_MIN_RTEXTSIZE)
return false;
if (xfs_sb_is_v5(sbp) &&
(sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_ZONED)) {
if (sbp->sb_rextsize != 1)
return false;
} else {
if (sbp->sb_rextsize * sbp->sb_blocksize > XFS_MAX_RTEXTSIZE ||
sbp->sb_rextsize * sbp->sb_blocksize < XFS_MIN_RTEXTSIZE)
return false;
}
if (sbp->sb_rblocks == 0) {
if (sbp->sb_rextents != 0 || sbp->sb_rbmblocks != 0 ||
@ -435,6 +446,34 @@ xfs_validate_sb_rtgroups(
return 0;
}
static int
xfs_validate_sb_zoned(
struct xfs_mount *mp,
struct xfs_sb *sbp)
{
if (sbp->sb_frextents != 0) {
xfs_warn(mp,
"sb_frextents must be zero for zoned file systems.");
return -EINVAL;
}
if (sbp->sb_rtstart && sbp->sb_rtstart < sbp->sb_dblocks) {
xfs_warn(mp,
"sb_rtstart (%lld) overlaps sb_dblocks (%lld).",
sbp->sb_rtstart, sbp->sb_dblocks);
return -EINVAL;
}
if (sbp->sb_rtreserved && sbp->sb_rtreserved >= sbp->sb_rblocks) {
xfs_warn(mp,
"sb_rtreserved (%lld) larger than sb_rblocks (%lld).",
sbp->sb_rtreserved, sbp->sb_rblocks);
return -EINVAL;
}
return 0;
}
/* Check the validity of the SB. */
STATIC int
xfs_validate_sb_common(
@ -523,6 +562,11 @@ xfs_validate_sb_common(
if (error)
return error;
}
if (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_ZONED) {
error = xfs_validate_sb_zoned(mp, sbp);
if (error)
return error;
}
} else if (sbp->sb_qflags & (XFS_PQUOTA_ENFD | XFS_GQUOTA_ENFD |
XFS_PQUOTA_CHKD | XFS_GQUOTA_CHKD)) {
xfs_notice(mp,
@ -835,6 +879,14 @@ __xfs_sb_from_disk(
to->sb_rgcount = 1;
to->sb_rgextents = 0;
}
if (to->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_ZONED) {
to->sb_rtstart = be64_to_cpu(from->sb_rtstart);
to->sb_rtreserved = be64_to_cpu(from->sb_rtreserved);
} else {
to->sb_rtstart = 0;
to->sb_rtreserved = 0;
}
}
void
@ -1001,6 +1053,11 @@ xfs_sb_to_disk(
to->sb_rbmino = cpu_to_be64(0);
to->sb_rsumino = cpu_to_be64(0);
}
if (from->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_ZONED) {
to->sb_rtstart = cpu_to_be64(from->sb_rtstart);
to->sb_rtreserved = cpu_to_be64(from->sb_rtreserved);
}
}
/*
@ -1146,6 +1203,10 @@ xfs_sb_mount_rextsize(
rgs->blocks = sbp->sb_rgextents * sbp->sb_rextsize;
rgs->blklog = mp->m_sb.sb_rgblklog;
rgs->blkmask = xfs_mask32lo(mp->m_sb.sb_rgblklog);
rgs->start_fsb = mp->m_sb.sb_rtstart;
if (xfs_sb_has_incompat_feature(sbp,
XFS_SB_FEAT_INCOMPAT_ZONE_GAPS))
rgs->has_daddr_gaps = true;
} else {
rgs->blocks = 0;
rgs->blklog = 0;
@ -1265,8 +1326,7 @@ xfs_log_sb(
mp->m_sb.sb_ifree = min_t(uint64_t,
percpu_counter_sum_positive(&mp->m_ifree),
mp->m_sb.sb_icount);
mp->m_sb.sb_fdblocks =
percpu_counter_sum_positive(&mp->m_fdblocks);
mp->m_sb.sb_fdblocks = xfs_sum_freecounter(mp, XC_FREE_BLOCKS);
}
/*
@ -1275,9 +1335,10 @@ xfs_log_sb(
* we handle nearly-lockless reservations, so we must use the _positive
* variant here to avoid writing out nonsense frextents.
*/
if (xfs_has_rtgroups(mp))
if (xfs_has_rtgroups(mp) && !xfs_has_zoned(mp)) {
mp->m_sb.sb_frextents =
percpu_counter_sum_positive(&mp->m_frextents);
xfs_sum_freecounter(mp, XC_FREE_RTEXTENTS);
}
xfs_sb_to_disk(bp->b_addr, &mp->m_sb);
xfs_trans_buf_set_type(tp, bp, XFS_BLFT_SB_BUF);
@ -1510,6 +1571,8 @@ xfs_fs_geometry(
geo->flags |= XFS_FSOP_GEOM_FLAGS_EXCHANGE_RANGE;
if (xfs_has_metadir(mp))
geo->flags |= XFS_FSOP_GEOM_FLAGS_METADIR;
if (xfs_has_zoned(mp))
geo->flags |= XFS_FSOP_GEOM_FLAGS_ZONED;
geo->rtsectsize = sbp->sb_blocksize;
geo->dirblocksize = xfs_dir2_dirblock_bytes(sbp);
@ -1530,6 +1593,10 @@ xfs_fs_geometry(
geo->rgcount = sbp->sb_rgcount;
geo->rgextents = sbp->sb_rgextents;
}
if (xfs_has_zoned(mp)) {
geo->rtstart = sbp->sb_rtstart;
geo->rtreserved = sbp->sb_rtreserved;
}
}
/* Read a secondary superblock. */

28
fs/xfs/libxfs/xfs_types.h
View File

@ -233,6 +233,34 @@ enum xfs_group_type {
{ XG_TYPE_AG, "ag" }, \
{ XG_TYPE_RTG, "rtg" }
enum xfs_free_counter {
/*
* Number of free blocks on the data device.
*/
XC_FREE_BLOCKS,
/*
* Number of free RT extents on the RT device.
*/
XC_FREE_RTEXTENTS,
/*
* Number of available for use RT extents.
*
* This counter only exists for zoned RT device and indicates the number
* of RT extents that can be directly used by writes. XC_FREE_RTEXTENTS
* also includes blocks that have been written previously and freed, but
* sit in a rtgroup that still needs a zone reset.
*/
XC_FREE_RTAVAILABLE,
XC_FREE_NR,
};
#define XFS_FREECOUNTER_STR \
{ XC_FREE_BLOCKS, "blocks" }, \
{ XC_FREE_RTEXTENTS, "rtextents" }, \
{ XC_FREE_RTAVAILABLE, "rtavailable" }
/*
* Type verifier functions
*/

186
fs/xfs/libxfs/xfs_zones.c Normal file
View File

@ -0,0 +1,186 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (c) 2023-2025 Christoph Hellwig.
* Copyright (c) 2024-2025, Western Digital Corporation or its affiliates.
*/
#include "xfs.h"
#include "xfs_fs.h"
#include "xfs_shared.h"
#include "xfs_format.h"
#include "xfs_log_format.h"
#include "xfs_trans_resv.h"
#include "xfs_mount.h"
#include "xfs_inode.h"
#include "xfs_rtgroup.h"
#include "xfs_zones.h"
static bool
xfs_zone_validate_empty(
struct blk_zone *zone,
struct xfs_rtgroup *rtg,
xfs_rgblock_t *write_pointer)
{
struct xfs_mount *mp = rtg_mount(rtg);
if (rtg_rmap(rtg)->i_used_blocks > 0) {
xfs_warn(mp, "empty zone %u has non-zero used counter (0x%x).",
rtg_rgno(rtg), rtg_rmap(rtg)->i_used_blocks);
return false;
}
*write_pointer = 0;
return true;
}
static bool
xfs_zone_validate_wp(
struct blk_zone *zone,
struct xfs_rtgroup *rtg,
xfs_rgblock_t *write_pointer)
{
struct xfs_mount *mp = rtg_mount(rtg);
xfs_rtblock_t wp_fsb = xfs_daddr_to_rtb(mp, zone->wp);
if (rtg_rmap(rtg)->i_used_blocks > rtg->rtg_extents) {
xfs_warn(mp, "zone %u has too large used counter (0x%x).",
rtg_rgno(rtg), rtg_rmap(rtg)->i_used_blocks);
return false;
}
if (xfs_rtb_to_rgno(mp, wp_fsb) != rtg_rgno(rtg)) {
xfs_warn(mp, "zone %u write pointer (0x%llx) outside of zone.",
rtg_rgno(rtg), wp_fsb);
return false;
}
*write_pointer = xfs_rtb_to_rgbno(mp, wp_fsb);
if (*write_pointer >= rtg->rtg_extents) {
xfs_warn(mp, "zone %u has invalid write pointer (0x%x).",
rtg_rgno(rtg), *write_pointer);
return false;
}
return true;
}
static bool
xfs_zone_validate_full(
struct blk_zone *zone,
struct xfs_rtgroup *rtg,
xfs_rgblock_t *write_pointer)
{
struct xfs_mount *mp = rtg_mount(rtg);
if (rtg_rmap(rtg)->i_used_blocks > rtg->rtg_extents) {
xfs_warn(mp, "zone %u has too large used counter (0x%x).",
rtg_rgno(rtg), rtg_rmap(rtg)->i_used_blocks);
return false;
}
*write_pointer = rtg->rtg_extents;
return true;
}
static bool
xfs_zone_validate_seq(
struct blk_zone *zone,
struct xfs_rtgroup *rtg,
xfs_rgblock_t *write_pointer)
{
struct xfs_mount *mp = rtg_mount(rtg);
switch (zone->cond) {
case BLK_ZONE_COND_EMPTY:
return xfs_zone_validate_empty(zone, rtg, write_pointer);
case BLK_ZONE_COND_IMP_OPEN:
case BLK_ZONE_COND_EXP_OPEN:
case BLK_ZONE_COND_CLOSED:
return xfs_zone_validate_wp(zone, rtg, write_pointer);
case BLK_ZONE_COND_FULL:
return xfs_zone_validate_full(zone, rtg, write_pointer);
case BLK_ZONE_COND_NOT_WP:
case BLK_ZONE_COND_OFFLINE:
case BLK_ZONE_COND_READONLY:
xfs_warn(mp, "zone %u has unsupported zone condition 0x%x.",
rtg_rgno(rtg), zone->cond);
return false;
default:
xfs_warn(mp, "zone %u has unknown zone condition 0x%x.",
rtg_rgno(rtg), zone->cond);
return false;
}
}
static bool
xfs_zone_validate_conv(
struct blk_zone *zone,
struct xfs_rtgroup *rtg)
{
struct xfs_mount *mp = rtg_mount(rtg);
switch (zone->cond) {
case BLK_ZONE_COND_NOT_WP:
return true;
default:
xfs_warn(mp,
"conventional zone %u has unsupported zone condition 0x%x.",
rtg_rgno(rtg), zone->cond);
return false;
}
}
bool
xfs_zone_validate(
struct blk_zone *zone,
struct xfs_rtgroup *rtg,
xfs_rgblock_t *write_pointer)
{
struct xfs_mount *mp = rtg_mount(rtg);
struct xfs_groups *g = &mp->m_groups[XG_TYPE_RTG];
uint32_t expected_size;
/*
* Check that the zone capacity matches the rtgroup size stored in the
* superblock. Note that all zones including the last one must have a
* uniform capacity.
*/
if (XFS_BB_TO_FSB(mp, zone->capacity) != g->blocks) {
xfs_warn(mp,
"zone %u capacity (0x%llx) does not match RT group size (0x%x).",
rtg_rgno(rtg), XFS_BB_TO_FSB(mp, zone->capacity),
g->blocks);
return false;
}
if (g->has_daddr_gaps) {
expected_size = 1 << g->blklog;
} else {
if (zone->len != zone->capacity) {
xfs_warn(mp,
"zone %u has capacity != size ((0x%llx vs 0x%llx)",
rtg_rgno(rtg),
XFS_BB_TO_FSB(mp, zone->len),
XFS_BB_TO_FSB(mp, zone->capacity));
return false;
}
expected_size = g->blocks;
}
if (XFS_BB_TO_FSB(mp, zone->len) != expected_size) {
xfs_warn(mp,
"zone %u length (0x%llx) does match geometry (0x%x).",
rtg_rgno(rtg), XFS_BB_TO_FSB(mp, zone->len),
expected_size);
}
switch (zone->type) {
case BLK_ZONE_TYPE_CONVENTIONAL:
return xfs_zone_validate_conv(zone, rtg);
case BLK_ZONE_TYPE_SEQWRITE_REQ:
return xfs_zone_validate_seq(zone, rtg, write_pointer);
default:
xfs_warn(mp, "zoned %u has unsupported type 0x%x.",
rtg_rgno(rtg), zone->type);
return false;
}
}

35
fs/xfs/libxfs/xfs_zones.h Normal file
View File

@ -0,0 +1,35 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LIBXFS_ZONES_H
#define _LIBXFS_ZONES_H
struct xfs_rtgroup;
/*
* In order to guarantee forward progress for GC we need to reserve at least
* two zones: one that will be used for moving data into and one spare zone
* making sure that we have enough space to relocate a nearly-full zone.
* To allow for slightly sloppy accounting for when we need to reserve the
* second zone, we actually reserve three as that is easier than doing fully
* accurate bookkeeping.
*/
#define XFS_GC_ZONES 3U
/*
* In addition we need two zones for user writes, one open zone for writing
* and one to still have available blocks without resetting the open zone
* when data in the open zone has been freed.
*/
#define XFS_RESERVED_ZONES (XFS_GC_ZONES + 1)
#define XFS_MIN_ZONES (XFS_RESERVED_ZONES + 1)
/*
* Always keep one zone out of the general open zone pool to allow for GC to
* happen while other writers are waiting for free space.
*/
#define XFS_OPEN_GC_ZONES 1U
#define XFS_MIN_OPEN_ZONES (XFS_OPEN_GC_ZONES + 1U)
bool xfs_zone_validate(struct blk_zone *zone, struct xfs_rtgroup *rtg,
xfs_rgblock_t *write_pointer);
#endif /* _LIBXFS_ZONES_H */

2
fs/xfs/scrub/agheader.c
View File

@ -69,6 +69,8 @@ STATIC size_t
xchk_superblock_ondisk_size(
struct xfs_mount *mp)
{
if (xfs_has_zoned(mp))
return offsetofend(struct xfs_dsb, sb_rtreserved);
if (xfs_has_metadir(mp))
return offsetofend(struct xfs_dsb, sb_pad);
if (xfs_has_metauuid(mp))

4
fs/xfs/scrub/bmap.c
View File

@ -1038,8 +1038,8 @@ xchk_bmap(
switch (whichfork) {
case XFS_COW_FORK:
/* No CoW forks on non-reflink filesystems. */
if (!xfs_has_reflink(mp)) {
/* No CoW forks filesystem doesn't support out of place writes */
if (!xfs_has_reflink(mp) && !xfs_has_zoned(mp)) {
xchk_ino_set_corrupt(sc, sc->ip->i_ino);
return 0;
}

22
fs/xfs/scrub/fscounters.c
View File

@ -350,7 +350,7 @@ xchk_fscount_aggregate_agcounts(
* The global incore space reservation is taken from the incore
* counters, so leave that out of the computation.
*/
fsc->fdblocks -= mp->m_resblks_avail;
fsc->fdblocks -= mp->m_free[XC_FREE_BLOCKS].res_avail;
/*
* Delayed allocation reservations are taken out of the incore counters
@ -413,7 +413,13 @@ xchk_fscount_count_frextents(
fsc->frextents = 0;
fsc->frextents_delayed = 0;
if (!xfs_has_realtime(mp))
/*
* Don't bother verifying and repairing the fs counters for zoned file
* systems as they don't track an on-disk frextents count, and the
* in-memory percpu counter also includes reservations.
*/
if (!xfs_has_realtime(mp) || xfs_has_zoned(mp))
return 0;
while ((rtg = xfs_rtgroup_next(mp, rtg))) {
@ -513,8 +519,8 @@ xchk_fscounters(
/* Snapshot the percpu counters. */
icount = percpu_counter_sum(&mp->m_icount);
ifree = percpu_counter_sum(&mp->m_ifree);
fdblocks = percpu_counter_sum(&mp->m_fdblocks);
frextents = percpu_counter_sum(&mp->m_frextents);
fdblocks = xfs_sum_freecounter_raw(mp, XC_FREE_BLOCKS);
frextents = xfs_sum_freecounter_raw(mp, XC_FREE_RTEXTENTS);
/* No negative values, please! */
if (icount < 0 || ifree < 0)
@ -589,15 +595,17 @@ xchk_fscounters(
try_again = true;
}
if (!xchk_fscount_within_range(sc, fdblocks, &mp->m_fdblocks,
fsc->fdblocks)) {
if (!xchk_fscount_within_range(sc, fdblocks,
&mp->m_free[XC_FREE_BLOCKS].count, fsc->fdblocks)) {
if (fsc->frozen)
xchk_set_corrupt(sc);
else
try_again = true;
}
if (!xchk_fscount_within_range(sc, frextents, &mp->m_frextents,
if (!xfs_has_zoned(mp) &&
!xchk_fscount_within_range(sc, frextents,
&mp->m_free[XC_FREE_RTEXTENTS].count,
fsc->frextents - fsc->frextents_delayed)) {
if (fsc->frozen)
xchk_set_corrupt(sc);

12
fs/xfs/scrub/fscounters_repair.c
View File

@ -64,7 +64,7 @@ xrep_fscounters(
percpu_counter_set(&mp->m_icount, fsc->icount);
percpu_counter_set(&mp->m_ifree, fsc->ifree);
percpu_counter_set(&mp->m_fdblocks, fsc->fdblocks);
xfs_set_freecounter(mp, XC_FREE_BLOCKS, fsc->fdblocks);
/*
* Online repair is only supported on v5 file systems, which require
@ -74,10 +74,12 @@ xrep_fscounters(
* track of the delalloc reservations separately, as they are are
* subtracted from m_frextents, but not included in sb_frextents.
*/
percpu_counter_set(&mp->m_frextents,
fsc->frextents - fsc->frextents_delayed);
if (!xfs_has_rtgroups(mp))
mp->m_sb.sb_frextents = fsc->frextents;
if (!xfs_has_zoned(mp)) {
xfs_set_freecounter(mp, XC_FREE_RTEXTENTS,
fsc->frextents - fsc->frextents_delayed);
if (!xfs_has_rtgroups(mp))
mp->m_sb.sb_frextents = fsc->frextents;
}
return 0;
}

7
fs/xfs/scrub/inode.c
View File

@ -273,6 +273,13 @@ xchk_inode_cowextsize(
xfs_failaddr_t fa;
uint32_t value = be32_to_cpu(dip->di_cowextsize);
/*
* The used block counter for rtrmap is checked and repaired elsewhere.
*/
if (xfs_has_zoned(sc->mp) &&
dip->di_metatype == cpu_to_be16(XFS_METAFILE_RTRMAP))
return;
fa = xfs_inode_validate_cowextsize(sc->mp, value, mode, flags, flags2);
if (fa)
xchk_ino_set_corrupt(sc, ino);

4
fs/xfs/scrub/inode_repair.c
View File

@ -710,7 +710,9 @@ xrep_dinode_extsize_hints(
XFS_DIFLAG_EXTSZINHERIT);
}
if (dip->di_version < 3)
if (dip->di_version < 3 ||
(xfs_has_zoned(sc->mp) &&
dip->di_metatype == cpu_to_be16(XFS_METAFILE_RTRMAP)))
return;
fa = xfs_inode_validate_cowextsize(mp, be32_to_cpu(dip->di_cowextsize),

2
fs/xfs/scrub/newbt.c
View File

@ -62,7 +62,7 @@ xrep_newbt_estimate_slack(
free = sc->sa.pag->pagf_freeblks;
sz = xfs_ag_block_count(sc->mp, pag_agno(sc->sa.pag));
} else {
free = percpu_counter_sum(&sc->mp->m_fdblocks);
free = xfs_sum_freecounter_raw(sc->mp, XC_FREE_BLOCKS);
sz = sc->mp->m_sb.sb_dblocks;
}

9
fs/xfs/scrub/reap.c
View File

@ -935,10 +935,13 @@ xrep_reap_metadir_fsblocks(
if (error)
return error;
if (xreap_dirty(&rs))
return xrep_defer_finish(sc);
if (xreap_dirty(&rs)) {
error = xrep_defer_finish(sc);
if (error)
return error;
}
return 0;
return xrep_reset_metafile_resv(sc);
}
/*

37
fs/xfs/scrub/repair.c
View File

@ -43,6 +43,7 @@
#include "xfs_rtalloc.h"
#include "xfs_metafile.h"
#include "xfs_rtrefcount_btree.h"
#include "xfs_zone_alloc.h"
#include "scrub/scrub.h"
#include "scrub/common.h"
#include "scrub/trace.h"
@ -1050,7 +1051,13 @@ xrep_require_rtext_inuse(
xfs_rtxnum_t startrtx;
xfs_rtxnum_t endrtx;
bool is_free = false;
int error;
int error = 0;
if (xfs_has_zoned(mp)) {
if (!xfs_zone_rgbno_is_valid(sc->sr.rtg, rgbno + len - 1))
return -EFSCORRUPTED;
return 0;
}
startrtx = xfs_rgbno_to_rtx(mp, rgbno);
endrtx = xfs_rgbno_to_rtx(mp, rgbno + len - 1);
@ -1386,11 +1393,12 @@ int
xrep_reset_metafile_resv(
struct xfs_scrub *sc)
{
struct xfs_inode *ip = sc->ip;
struct xfs_mount *mp = sc->mp;
int64_t delta;
int error;
delta = ip->i_nblocks + ip->i_delayed_blks - ip->i_meta_resv_asked;
delta = mp->m_metafile_resv_used + mp->m_metafile_resv_avail -
mp->m_metafile_resv_target;
if (delta == 0)
return 0;
@ -1401,11 +1409,11 @@ xrep_reset_metafile_resv(
if (delta > 0) {
int64_t give_back;
give_back = min_t(uint64_t, delta, ip->i_delayed_blks);
give_back = min_t(uint64_t, delta, mp->m_metafile_resv_avail);
if (give_back > 0) {
xfs_mod_delalloc(ip, 0, -give_back);
xfs_add_fdblocks(ip->i_mount, give_back);
ip->i_delayed_blks -= give_back;
xfs_mod_sb_delalloc(mp, -give_back);
xfs_add_fdblocks(mp, give_back);
mp->m_metafile_resv_avail -= give_back;
}
return 0;
@ -1413,24 +1421,23 @@ xrep_reset_metafile_resv(
/*
* Not enough reservation; try to take some blocks from the filesystem
* to the metadata inode. @delta is negative here, so invert the sign.
* to the metabtree reservation.
*/
delta = -delta;
error = xfs_dec_fdblocks(sc->mp, delta, true);
delta = -delta; /* delta is negative here, so invert the sign. */
error = xfs_dec_fdblocks(mp, delta, true);
while (error == -ENOSPC) {
delta--;
if (delta == 0) {
xfs_warn(sc->mp,
"Insufficient free space to reset space reservation for inode 0x%llx after repair.",
ip->i_ino);
"Insufficient free space to reset metabtree reservation after repair.");
return 0;
}
error = xfs_dec_fdblocks(sc->mp, delta, true);
error = xfs_dec_fdblocks(mp, delta, true);
}
if (error)
return error;
xfs_mod_delalloc(ip, 0, delta);
ip->i_delayed_blks += delta;
xfs_mod_sb_delalloc(mp, delta);
mp->m_metafile_resv_avail += delta;
return 0;
}

11
fs/xfs/scrub/rtbitmap.c
View File

@ -21,6 +21,7 @@
#include "xfs_rmap.h"
#include "xfs_rtrmap_btree.h"
#include "xfs_exchmaps.h"
#include "xfs_zone_alloc.h"
#include "scrub/scrub.h"
#include "scrub/common.h"
#include "scrub/repair.h"
@ -272,7 +273,6 @@ xchk_xref_is_used_rt_space(
xfs_extlen_t len)
{
struct xfs_rtgroup *rtg = sc->sr.rtg;
struct xfs_inode *rbmip = rtg_bitmap(rtg);
xfs_rtxnum_t startext;
xfs_rtxnum_t endext;
bool is_free;
@ -281,6 +281,13 @@ xchk_xref_is_used_rt_space(
if (xchk_skip_xref(sc->sm))
return;
if (xfs_has_zoned(sc->mp)) {
if (!xfs_zone_rgbno_is_valid(rtg,
xfs_rtb_to_rgbno(sc->mp, rtbno) + len - 1))
xchk_ino_xref_set_corrupt(sc, rtg_rmap(rtg)->i_ino);
return;
}
startext = xfs_rtb_to_rtx(sc->mp, rtbno);
endext = xfs_rtb_to_rtx(sc->mp, rtbno + len - 1);
error = xfs_rtalloc_extent_is_free(rtg, sc->tp, startext,
@ -288,5 +295,5 @@ xchk_xref_is_used_rt_space(
if (!xchk_should_check_xref(sc, &error, NULL))
return;
if (is_free)
xchk_ino_xref_set_corrupt(sc, rbmip->i_ino);
xchk_ino_xref_set_corrupt(sc, rtg_bitmap(rtg)->i_ino);
}

34
fs/xfs/scrub/rtrefcount_repair.c
View File

@ -697,32 +697,6 @@ xrep_rtrefc_build_new_tree(
return error;
}
/*
* Now that we've logged the roots of the new btrees, invalidate all of the
* old blocks and free them.
*/
STATIC int
xrep_rtrefc_remove_old_tree(
struct xrep_rtrefc *rr)
{
int error;
/*
* Free all the extents that were allocated to the former rtrefcountbt
* and aren't cross-linked with something else.
*/
error = xrep_reap_metadir_fsblocks(rr->sc,
&rr->old_rtrefcountbt_blocks);
if (error)
return error;
/*
* Ensure the proper reservation for the rtrefcount inode so that we
* don't fail to expand the btree.
*/
return xrep_reset_metafile_resv(rr->sc);
}
/* Rebuild the rt refcount btree. */
int
xrep_rtrefcountbt(
@ -769,8 +743,12 @@ xrep_rtrefcountbt(
if (error)
goto out_bitmap;
/* Kill the old tree. */
error = xrep_rtrefc_remove_old_tree(rr);
/*
* Free all the extents that were allocated to the former rtrefcountbt
* and aren't cross-linked with something else.
*/
error = xrep_reap_metadir_fsblocks(rr->sc,
&rr->old_rtrefcountbt_blocks);
if (error)
goto out_bitmap;

29
fs/xfs/scrub/rtrmap_repair.c
View File

@ -810,28 +810,6 @@ xrep_rtrmap_build_new_tree(
/* Reaping the old btree. */
/* Reap the old rtrmapbt blocks. */
STATIC int
xrep_rtrmap_remove_old_tree(
struct xrep_rtrmap *rr)
{
int error;
/*
* Free all the extents that were allocated to the former rtrmapbt and
* aren't cross-linked with something else.
*/
error = xrep_reap_metadir_fsblocks(rr->sc, &rr->old_rtrmapbt_blocks);
if (error)
return error;
/*
* Ensure the proper reservation for the rtrmap inode so that we don't
* fail to expand the new btree.
*/
return xrep_reset_metafile_resv(rr->sc);
}
static inline bool
xrep_rtrmapbt_want_live_update(
struct xchk_iscan *iscan,
@ -995,8 +973,11 @@ xrep_rtrmapbt(
if (error)
goto out_records;
/* Kill the old tree. */
error = xrep_rtrmap_remove_old_tree(rr);
/*
* Free all the extents that were allocated to the former rtrmapbt and
* aren't cross-linked with something else.
*/
error = xrep_reap_metadir_fsblocks(rr->sc, &rr->old_rtrmapbt_blocks);
if (error)
goto out_records;

2
fs/xfs/scrub/scrub.c
View File

@ -399,12 +399,14 @@ static const struct xchk_meta_ops meta_scrub_ops[] = {
},
[XFS_SCRUB_TYPE_RTBITMAP] = { /* realtime bitmap */
.type = ST_RTGROUP,
.has = xfs_has_nonzoned,
.setup = xchk_setup_rtbitmap,
.scrub = xchk_rtbitmap,
.repair = xrep_rtbitmap,
},
[XFS_SCRUB_TYPE_RTSUM] = { /* realtime summary */
.type = ST_RTGROUP,
.has = xfs_has_nonzoned,
.setup = xchk_setup_rtsummary,
.scrub = xchk_rtsummary,
.repair = xrep_rtsummary,

194
fs/xfs/xfs_aops.c
View File

@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (c) 2000-2005 Silicon Graphics, Inc.
* Copyright (c) 2016-2018 Christoph Hellwig.
* Copyright (c) 2016-2025 Christoph Hellwig.
* All Rights Reserved.
*/
#include "xfs.h"
@ -20,6 +20,8 @@
#include "xfs_errortag.h"
#include "xfs_error.h"
#include "xfs_icache.h"
#include "xfs_zone_alloc.h"
#include "xfs_rtgroup.h"
struct xfs_writepage_ctx {
struct iomap_writepage_ctx ctx;
@ -77,6 +79,26 @@ xfs_setfilesize(
return xfs_trans_commit(tp);
}
static void
xfs_ioend_put_open_zones(
struct iomap_ioend *ioend)
{
struct iomap_ioend *tmp;
/*
* Put the open zone for all ioends merged into this one (if any).
*/
list_for_each_entry(tmp, &ioend->io_list, io_list)
xfs_open_zone_put(tmp->io_private);
/*
* The main ioend might not have an open zone if the submission failed
* before xfs_zone_alloc_and_submit got called.
*/
if (ioend->io_private)
xfs_open_zone_put(ioend->io_private);
}
/*
* IO write completion.
*/
@ -86,6 +108,7 @@ xfs_end_ioend(
{
struct xfs_inode *ip = XFS_I(ioend->io_inode);
struct xfs_mount *mp = ip->i_mount;
bool is_zoned = xfs_is_zoned_inode(ip);
xfs_off_t offset = ioend->io_offset;
size_t size = ioend->io_size;
unsigned int nofs_flag;
@ -115,10 +138,11 @@ xfs_end_ioend(
*/
error = blk_status_to_errno(ioend->io_bio.bi_status);
if (unlikely(error)) {
if (ioend->io_flags & IOMAP_F_SHARED) {
if (ioend->io_flags & IOMAP_IOEND_SHARED) {
ASSERT(!is_zoned);
xfs_reflink_cancel_cow_range(ip, offset, size, true);
xfs_bmap_punch_delalloc_range(ip, XFS_DATA_FORK, offset,
offset + size);
offset + size, NULL);
}
goto done;
}
@ -126,14 +150,21 @@ xfs_end_ioend(
/*
* Success: commit the COW or unwritten blocks if needed.
*/
if (ioend->io_flags & IOMAP_F_SHARED)
if (is_zoned)
error = xfs_zoned_end_io(ip, offset, size, ioend->io_sector,
ioend->io_private, NULLFSBLOCK);
else if (ioend->io_flags & IOMAP_IOEND_SHARED)
error = xfs_reflink_end_cow(ip, offset, size);
else if (ioend->io_type == IOMAP_UNWRITTEN)
else if (ioend->io_flags & IOMAP_IOEND_UNWRITTEN)
error = xfs_iomap_write_unwritten(ip, offset, size, false);
if (!error && xfs_ioend_is_append(ioend))
if (!error &&
!(ioend->io_flags & IOMAP_IOEND_DIRECT) &&
xfs_ioend_is_append(ioend))
error = xfs_setfilesize(ip, offset, size);
done:
if (is_zoned)
xfs_ioend_put_open_zones(ioend);
iomap_finish_ioends(ioend, error);
memalloc_nofs_restore(nofs_flag);
}
@ -176,17 +207,27 @@ xfs_end_io(
}
}
STATIC void
void
xfs_end_bio(
struct bio *bio)
{
struct iomap_ioend *ioend = iomap_ioend_from_bio(bio);
struct xfs_inode *ip = XFS_I(ioend->io_inode);
struct xfs_mount *mp = ip->i_mount;
unsigned long flags;
/*
* For Appends record the actually written block number and set the
* boundary flag if needed.
*/
if (IS_ENABLED(CONFIG_XFS_RT) && bio_is_zone_append(bio)) {
ioend->io_sector = bio->bi_iter.bi_sector;
xfs_mark_rtg_boundary(ioend);
}
spin_lock_irqsave(&ip->i_ioend_lock, flags);
if (list_empty(&ip->i_ioend_list))
WARN_ON_ONCE(!queue_work(ip->i_mount->m_unwritten_workqueue,
WARN_ON_ONCE(!queue_work(mp->m_unwritten_workqueue,
&ip->i_ioend_work));
list_add_tail(&ioend->io_list, &ip->i_ioend_list);
spin_unlock_irqrestore(&ip->i_ioend_lock, flags);
@ -396,10 +437,11 @@ xfs_map_blocks(
}
static int
xfs_prepare_ioend(
struct iomap_ioend *ioend,
xfs_submit_ioend(
struct iomap_writepage_ctx *wpc,
int status)
{
struct iomap_ioend *ioend = wpc->ioend;
unsigned int nofs_flag;
/*
@ -410,7 +452,7 @@ xfs_prepare_ioend(
nofs_flag = memalloc_nofs_save();
/* Convert CoW extents to regular */
if (!status && (ioend->io_flags & IOMAP_F_SHARED)) {
if (!status && (ioend->io_flags & IOMAP_IOEND_SHARED)) {
status = xfs_reflink_convert_cow(XFS_I(ioend->io_inode),
ioend->io_offset, ioend->io_size);
}
@ -418,10 +460,14 @@ xfs_prepare_ioend(
memalloc_nofs_restore(nofs_flag);
/* send ioends that might require a transaction to the completion wq */
if (xfs_ioend_is_append(ioend) || ioend->io_type == IOMAP_UNWRITTEN ||
(ioend->io_flags & IOMAP_F_SHARED))
if (xfs_ioend_is_append(ioend) ||
(ioend->io_flags & (IOMAP_IOEND_UNWRITTEN | IOMAP_IOEND_SHARED)))
ioend->io_bio.bi_end_io = xfs_end_bio;
return status;
if (status)
return status;
submit_bio(&ioend->io_bio);
return 0;
}
/*
@ -458,12 +504,107 @@ xfs_discard_folio(
* folio itself and not the start offset that is passed in.
*/
xfs_bmap_punch_delalloc_range(ip, XFS_DATA_FORK, pos,
folio_pos(folio) + folio_size(folio));
folio_pos(folio) + folio_size(folio), NULL);
}
static const struct iomap_writeback_ops xfs_writeback_ops = {
.map_blocks = xfs_map_blocks,
.prepare_ioend = xfs_prepare_ioend,
.submit_ioend = xfs_submit_ioend,
.discard_folio = xfs_discard_folio,
};
struct xfs_zoned_writepage_ctx {
struct iomap_writepage_ctx ctx;
struct xfs_open_zone *open_zone;
};
static inline struct xfs_zoned_writepage_ctx *
XFS_ZWPC(struct iomap_writepage_ctx *ctx)
{
return container_of(ctx, struct xfs_zoned_writepage_ctx, ctx);
}
static int
xfs_zoned_map_blocks(
struct iomap_writepage_ctx *wpc,
struct inode *inode,
loff_t offset,
unsigned int len)
{
struct xfs_inode *ip = XFS_I(inode);
struct xfs_mount *mp = ip->i_mount;
xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset);
xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + len);
xfs_filblks_t count_fsb;
struct xfs_bmbt_irec imap, del;
struct xfs_iext_cursor icur;
if (xfs_is_shutdown(mp))
return -EIO;
XFS_ERRORTAG_DELAY(mp, XFS_ERRTAG_WB_DELAY_MS);
/*
* All dirty data must be covered by delalloc extents. But truncate can
* remove delalloc extents underneath us or reduce their size.
* Returning a hole tells iomap to not write back any data from this
* range, which is the right thing to do in that case.
*
* Otherwise just tell iomap to treat ranges previously covered by a
* delalloc extent as mapped. The actual block allocation will be done
* just before submitting the bio.
*
* This implies we never map outside folios that are locked or marked
* as under writeback, and thus there is no need check the fork sequence
* count here.
*/
xfs_ilock(ip, XFS_ILOCK_EXCL);
if (!xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &imap))
imap.br_startoff = end_fsb; /* fake a hole past EOF */
if (imap.br_startoff > offset_fsb) {
imap.br_blockcount = imap.br_startoff - offset_fsb;
imap.br_startoff = offset_fsb;
imap.br_startblock = HOLESTARTBLOCK;
imap.br_state = XFS_EXT_NORM;
xfs_iunlock(ip, XFS_ILOCK_EXCL);
xfs_bmbt_to_iomap(ip, &wpc->iomap, &imap, 0, 0, 0);
return 0;
}
end_fsb = min(end_fsb, imap.br_startoff + imap.br_blockcount);
count_fsb = end_fsb - offset_fsb;
del = imap;
xfs_trim_extent(&del, offset_fsb, count_fsb);
xfs_bmap_del_extent_delay(ip, XFS_COW_FORK, &icur, &imap, &del,
XFS_BMAPI_REMAP);
xfs_iunlock(ip, XFS_ILOCK_EXCL);
wpc->iomap.type = IOMAP_MAPPED;
wpc->iomap.flags = IOMAP_F_DIRTY;
wpc->iomap.bdev = mp->m_rtdev_targp->bt_bdev;
wpc->iomap.offset = offset;
wpc->iomap.length = XFS_FSB_TO_B(mp, count_fsb);
wpc->iomap.flags = IOMAP_F_ANON_WRITE;
trace_xfs_zoned_map_blocks(ip, offset, wpc->iomap.length);
return 0;
}
static int
xfs_zoned_submit_ioend(
struct iomap_writepage_ctx *wpc,
int status)
{
wpc->ioend->io_bio.bi_end_io = xfs_end_bio;
if (status)
return status;
xfs_zone_alloc_and_submit(wpc->ioend, &XFS_ZWPC(wpc)->open_zone);
return 0;
}
static const struct iomap_writeback_ops xfs_zoned_writeback_ops = {
.map_blocks = xfs_zoned_map_blocks,
.submit_ioend = xfs_zoned_submit_ioend,
.discard_folio = xfs_discard_folio,
};
@ -472,10 +613,25 @@ xfs_vm_writepages(
struct address_space *mapping,
struct writeback_control *wbc)
{
struct xfs_writepage_ctx wpc = { };
struct xfs_inode *ip = XFS_I(mapping->host);
xfs_iflags_clear(XFS_I(mapping->host), XFS_ITRUNCATED);
return iomap_writepages(mapping, wbc, &wpc.ctx, &xfs_writeback_ops);
xfs_iflags_clear(ip, XFS_ITRUNCATED);
if (xfs_is_zoned_inode(ip)) {
struct xfs_zoned_writepage_ctx xc = { };
int error;
error = iomap_writepages(mapping, wbc, &xc.ctx,
&xfs_zoned_writeback_ops);
if (xc.open_zone)
xfs_open_zone_put(xc.open_zone);
return error;
} else {
struct xfs_writepage_ctx wpc = { };
return iomap_writepages(mapping, wbc, &wpc.ctx,
&xfs_writeback_ops);
}
}
STATIC int

3
fs/xfs/xfs_aops.h
View File

@ -9,6 +9,7 @@
extern const struct address_space_operations xfs_address_space_operations;
extern const struct address_space_operations xfs_dax_aops;
int xfs_setfilesize(struct xfs_inode *ip, xfs_off_t offset, size_t size);
int xfs_setfilesize(struct xfs_inode *ip, xfs_off_t offset, size_t size);
void xfs_end_bio(struct bio *bio);
#endif /* __XFS_AOPS_H__ */

32
fs/xfs/xfs_bmap_util.c
View File

@ -30,6 +30,7 @@
#include "xfs_reflink.h"
#include "xfs_rtbitmap.h"
#include "xfs_rtgroup.h"
#include "xfs_zone_alloc.h"
/* Kernel only BMAP related definitions and functions */
@ -436,7 +437,8 @@ xfs_bmap_punch_delalloc_range(
struct xfs_inode *ip,
int whichfork,
xfs_off_t start_byte,
xfs_off_t end_byte)
xfs_off_t end_byte,
struct xfs_zone_alloc_ctx *ac)
{
struct xfs_mount *mp = ip->i_mount;
struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork);
@ -467,7 +469,21 @@ xfs_bmap_punch_delalloc_range(
continue;
}
xfs_bmap_del_extent_delay(ip, whichfork, &icur, &got, &del);
if (xfs_is_zoned_inode(ip) && ac) {
/*
* In a zoned buffered write context we need to return
* the punched delalloc allocations to the allocation
* context. This allows reusing them in the following
* iomap iterations.
*/
xfs_bmap_del_extent_delay(ip, whichfork, &icur, &got,
&del, XFS_BMAPI_REMAP);
ac->reserved_blocks += del.br_blockcount;
} else {
xfs_bmap_del_extent_delay(ip, whichfork, &icur, &got,
&del, 0);
}
if (!xfs_iext_get_extent(ifp, &icur, &got))
break;
}
@ -582,7 +598,7 @@ xfs_free_eofblocks(
if (ip->i_delayed_blks) {
xfs_bmap_punch_delalloc_range(ip, XFS_DATA_FORK,
round_up(XFS_ISIZE(ip), mp->m_sb.sb_blocksize),
LLONG_MAX);
LLONG_MAX, NULL);
}
xfs_inode_clear_eofblocks_tag(ip);
return 0;
@ -825,7 +841,8 @@ int
xfs_free_file_space(
struct xfs_inode *ip,
xfs_off_t offset,
xfs_off_t len)
xfs_off_t len,
struct xfs_zone_alloc_ctx *ac)
{
struct xfs_mount *mp = ip->i_mount;
xfs_fileoff_t startoffset_fsb;
@ -880,7 +897,7 @@ xfs_free_file_space(
return 0;
if (offset + len > XFS_ISIZE(ip))
len = XFS_ISIZE(ip) - offset;
error = xfs_zero_range(ip, offset, len, NULL);
error = xfs_zero_range(ip, offset, len, ac, NULL);
if (error)
return error;
@ -968,7 +985,8 @@ int
xfs_collapse_file_space(
struct xfs_inode *ip,
xfs_off_t offset,
xfs_off_t len)
xfs_off_t len,
struct xfs_zone_alloc_ctx *ac)
{
struct xfs_mount *mp = ip->i_mount;
struct xfs_trans *tp;
@ -981,7 +999,7 @@ xfs_collapse_file_space(
trace_xfs_collapse_file_space(ip);
error = xfs_free_file_space(ip, offset, len);
error = xfs_free_file_space(ip, offset, len, ac);
if (error)
return error;

12
fs/xfs/xfs_bmap_util.h
View File

@ -15,6 +15,7 @@ struct xfs_inode;
struct xfs_mount;
struct xfs_trans;
struct xfs_bmalloca;
struct xfs_zone_alloc_ctx;
#ifdef CONFIG_XFS_RT
int xfs_bmap_rtalloc(struct xfs_bmalloca *ap);
@ -31,7 +32,8 @@ xfs_bmap_rtalloc(struct xfs_bmalloca *ap)
#endif /* CONFIG_XFS_RT */
void xfs_bmap_punch_delalloc_range(struct xfs_inode *ip, int whichfork,
xfs_off_t start_byte, xfs_off_t end_byte);
xfs_off_t start_byte, xfs_off_t end_byte,
struct xfs_zone_alloc_ctx *ac);
struct kgetbmap {
__s64 bmv_offset; /* file offset of segment in blocks */
@ -54,13 +56,13 @@ int xfs_bmap_last_extent(struct xfs_trans *tp, struct xfs_inode *ip,
/* preallocation and hole punch interface */
int xfs_alloc_file_space(struct xfs_inode *ip, xfs_off_t offset,
xfs_off_t len);
xfs_off_t len);
int xfs_free_file_space(struct xfs_inode *ip, xfs_off_t offset,
xfs_off_t len);
xfs_off_t len, struct xfs_zone_alloc_ctx *ac);
int xfs_collapse_file_space(struct xfs_inode *, xfs_off_t offset,
xfs_off_t len);
xfs_off_t len, struct xfs_zone_alloc_ctx *ac);
int xfs_insert_file_space(struct xfs_inode *, xfs_off_t offset,
xfs_off_t len);
xfs_off_t len);
/* EOF block manipulation functions */
bool xfs_can_free_eofblocks(struct xfs_inode *ip);

3
fs/xfs/xfs_discard.c
View File

@ -844,7 +844,8 @@ xfs_ioc_trim(
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
if (mp->m_rtdev_targp &&
if (mp->m_rtdev_targp && !xfs_has_zoned(mp) &&
bdev_max_discard_sectors(mp->m_rtdev_targp->bt_bdev))
rt_bdev = mp->m_rtdev_targp->bt_bdev;
if (!bdev_max_discard_sectors(mp->m_ddev_targp->bt_bdev) && !rt_bdev)

2
fs/xfs/xfs_extent_busy.c
View File

@ -671,7 +671,7 @@ xfs_extent_busy_wait_all(
while ((pag = xfs_perag_next(mp, pag)))
xfs_extent_busy_wait_group(pag_group(pag));
if (xfs_has_rtgroups(mp))
if (xfs_has_rtgroups(mp) && !xfs_has_zoned(mp))
while ((rtg = xfs_rtgroup_next(mp, rtg)))
xfs_extent_busy_wait_group(rtg_group(rtg));
}

35
fs/xfs/xfs_extfree_item.c
View File

@ -29,6 +29,7 @@
#include "xfs_inode.h"
#include "xfs_rtbitmap.h"
#include "xfs_rtgroup.h"
#include "xfs_zone_alloc.h"
struct kmem_cache *xfs_efi_cache;
struct kmem_cache *xfs_efd_cache;
@ -767,21 +768,35 @@ xfs_rtextent_free_finish_item(
trace_xfs_extent_free_deferred(mp, xefi);
if (!(xefi->xefi_flags & XFS_EFI_CANCELLED)) {
if (*rtgp != to_rtg(xefi->xefi_group)) {
*rtgp = to_rtg(xefi->xefi_group);
xfs_rtgroup_lock(*rtgp, XFS_RTGLOCK_BITMAP);
xfs_rtgroup_trans_join(tp, *rtgp,
XFS_RTGLOCK_BITMAP);
}
error = xfs_rtfree_blocks(tp, *rtgp,
xefi->xefi_startblock, xefi->xefi_blockcount);
if (xefi->xefi_flags & XFS_EFI_CANCELLED)
goto done;
if (*rtgp != to_rtg(xefi->xefi_group)) {
unsigned int lock_flags;
if (xfs_has_zoned(mp))
lock_flags = XFS_RTGLOCK_RMAP;
else
lock_flags = XFS_RTGLOCK_BITMAP;
*rtgp = to_rtg(xefi->xefi_group);
xfs_rtgroup_lock(*rtgp, lock_flags);
xfs_rtgroup_trans_join(tp, *rtgp, lock_flags);
}
if (xfs_has_zoned(mp)) {
error = xfs_zone_free_blocks(tp, *rtgp, xefi->xefi_startblock,
xefi->xefi_blockcount);
} else {
error = xfs_rtfree_blocks(tp, *rtgp, xefi->xefi_startblock,
xefi->xefi_blockcount);
}
if (error == -EAGAIN) {
xfs_efd_from_efi(efdp);
return error;
}
done:
xfs_efd_add_extent(efdp, xefi);
xfs_extent_free_cancel_item(item);
return error;

351
fs/xfs/xfs_file.c
View File

@ -25,6 +25,8 @@
#include "xfs_iomap.h"
#include "xfs_reflink.h"
#include "xfs_file.h"
#include "xfs_aops.h"
#include "xfs_zone_alloc.h"
#include <linux/dax.h>
#include <linux/falloc.h>
@ -150,7 +152,7 @@ xfs_file_fsync(
* ensure newly written file data make it to disk before logging the new
* inode size in case of an extending write.
*/
if (XFS_IS_REALTIME_INODE(ip))
if (XFS_IS_REALTIME_INODE(ip) && mp->m_rtdev_targp != mp->m_ddev_targp)
error = blkdev_issue_flush(mp->m_rtdev_targp->bt_bdev);
else if (mp->m_logdev_targp != mp->m_ddev_targp)
error = blkdev_issue_flush(mp->m_ddev_targp->bt_bdev);
@ -360,7 +362,8 @@ xfs_file_write_zero_eof(
struct iov_iter *from,
unsigned int *iolock,
size_t count,
bool *drained_dio)
bool *drained_dio,
struct xfs_zone_alloc_ctx *ac)
{
struct xfs_inode *ip = XFS_I(iocb->ki_filp->f_mapping->host);
loff_t isize;
@ -414,7 +417,7 @@ xfs_file_write_zero_eof(
trace_xfs_zero_eof(ip, isize, iocb->ki_pos - isize);
xfs_ilock(ip, XFS_MMAPLOCK_EXCL);
error = xfs_zero_range(ip, isize, iocb->ki_pos - isize, NULL);
error = xfs_zero_range(ip, isize, iocb->ki_pos - isize, ac, NULL);
xfs_iunlock(ip, XFS_MMAPLOCK_EXCL);
return error;
@ -431,7 +434,8 @@ STATIC ssize_t
xfs_file_write_checks(
struct kiocb *iocb,
struct iov_iter *from,
unsigned int *iolock)
unsigned int *iolock,
struct xfs_zone_alloc_ctx *ac)
{
struct inode *inode = iocb->ki_filp->f_mapping->host;
size_t count = iov_iter_count(from);
@ -481,7 +485,7 @@ xfs_file_write_checks(
*/
if (iocb->ki_pos > i_size_read(inode)) {
error = xfs_file_write_zero_eof(iocb, from, iolock, count,
&drained_dio);
&drained_dio, ac);
if (error == 1)
goto restart;
if (error)
@ -491,6 +495,48 @@ xfs_file_write_checks(
return kiocb_modified(iocb);
}
static ssize_t
xfs_zoned_write_space_reserve(
struct xfs_inode *ip,
struct kiocb *iocb,
struct iov_iter *from,
unsigned int flags,
struct xfs_zone_alloc_ctx *ac)
{
loff_t count = iov_iter_count(from);
int error;
if (iocb->ki_flags & IOCB_NOWAIT)
flags |= XFS_ZR_NOWAIT;
/*
* Check the rlimit and LFS boundary first so that we don't over-reserve
* by possibly a lot.
*
* The generic write path will redo this check later, and it might have
* changed by then. If it got expanded we'll stick to our earlier
* smaller limit, and if it is decreased the new smaller limit will be
* used and our extra space reservation will be returned after finishing
* the write.
*/
error = generic_write_check_limits(iocb->ki_filp, iocb->ki_pos, &count);
if (error)
return error;
/*
* Sloppily round up count to file system blocks.
*
* This will often reserve an extra block, but that avoids having to look
* at the start offset, which isn't stable for O_APPEND until taking the
* iolock. Also we need to reserve a block each for zeroing the old
* EOF block and the new start block if they are unaligned.
*
* Any remaining block will be returned after the write.
*/
return xfs_zoned_space_reserve(ip,
XFS_B_TO_FSB(ip->i_mount, count) + 1 + 2, flags, ac);
}
static int
xfs_dio_write_end_io(
struct kiocb *iocb,
@ -503,6 +549,9 @@ xfs_dio_write_end_io(
loff_t offset = iocb->ki_pos;
unsigned int nofs_flag;
ASSERT(!xfs_is_zoned_inode(ip) ||
!(flags & (IOMAP_DIO_UNWRITTEN | IOMAP_DIO_COW)));
trace_xfs_end_io_direct_write(ip, offset, size);
if (xfs_is_shutdown(ip->i_mount))
@ -582,14 +631,51 @@ static const struct iomap_dio_ops xfs_dio_write_ops = {
.end_io = xfs_dio_write_end_io,
};
static void
xfs_dio_zoned_submit_io(
const struct iomap_iter *iter,
struct bio *bio,
loff_t file_offset)
{
struct xfs_mount *mp = XFS_I(iter->inode)->i_mount;
struct xfs_zone_alloc_ctx *ac = iter->private;
xfs_filblks_t count_fsb;
struct iomap_ioend *ioend;
count_fsb = XFS_B_TO_FSB(mp, bio->bi_iter.bi_size);
if (count_fsb > ac->reserved_blocks) {
xfs_err(mp,
"allocation (%lld) larger than reservation (%lld).",
count_fsb, ac->reserved_blocks);
xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE);
bio_io_error(bio);
return;
}
ac->reserved_blocks -= count_fsb;
bio->bi_end_io = xfs_end_bio;
ioend = iomap_init_ioend(iter->inode, bio, file_offset,
IOMAP_IOEND_DIRECT);
xfs_zone_alloc_and_submit(ioend, &ac->open_zone);
}
static const struct iomap_dio_ops xfs_dio_zoned_write_ops = {
.bio_set = &iomap_ioend_bioset,
.submit_io = xfs_dio_zoned_submit_io,
.end_io = xfs_dio_write_end_io,
};
/*
* Handle block aligned direct I/O writes
* Handle block aligned direct I/O writes.
*/
static noinline ssize_t
xfs_file_dio_write_aligned(
struct xfs_inode *ip,
struct kiocb *iocb,
struct iov_iter *from)
struct iov_iter *from,
const struct iomap_ops *ops,
const struct iomap_dio_ops *dops,
struct xfs_zone_alloc_ctx *ac)
{
unsigned int iolock = XFS_IOLOCK_SHARED;
ssize_t ret;
@ -597,7 +683,7 @@ xfs_file_dio_write_aligned(
ret = xfs_ilock_iocb_for_write(iocb, &iolock);
if (ret)
return ret;
ret = xfs_file_write_checks(iocb, from, &iolock);
ret = xfs_file_write_checks(iocb, from, &iolock, ac);
if (ret)
goto out_unlock;
@ -611,11 +697,31 @@ xfs_file_dio_write_aligned(
iolock = XFS_IOLOCK_SHARED;
}
trace_xfs_file_direct_write(iocb, from);
ret = iomap_dio_rw(iocb, from, &xfs_direct_write_iomap_ops,
&xfs_dio_write_ops, 0, NULL, 0);
ret = iomap_dio_rw(iocb, from, ops, dops, 0, ac, 0);
out_unlock:
if (iolock)
xfs_iunlock(ip, iolock);
xfs_iunlock(ip, iolock);
return ret;
}
/*
* Handle block aligned direct I/O writes to zoned devices.
*/
static noinline ssize_t
xfs_file_dio_write_zoned(
struct xfs_inode *ip,
struct kiocb *iocb,
struct iov_iter *from)
{
struct xfs_zone_alloc_ctx ac = { };
ssize_t ret;
ret = xfs_zoned_write_space_reserve(ip, iocb, from, 0, &ac);
if (ret < 0)
return ret;
ret = xfs_file_dio_write_aligned(ip, iocb, from,
&xfs_zoned_direct_write_iomap_ops,
&xfs_dio_zoned_write_ops, &ac);
xfs_zoned_space_unreserve(ip, &ac);
return ret;
}
@ -675,7 +781,7 @@ xfs_file_dio_write_unaligned(
goto out_unlock;
}
ret = xfs_file_write_checks(iocb, from, &iolock);
ret = xfs_file_write_checks(iocb, from, &iolock, NULL);
if (ret)
goto out_unlock;
@ -721,9 +827,21 @@ xfs_file_dio_write(
/* direct I/O must be aligned to device logical sector size */
if ((iocb->ki_pos | count) & target->bt_logical_sectormask)
return -EINVAL;
if ((iocb->ki_pos | count) & ip->i_mount->m_blockmask)
/*
* For always COW inodes we also must check the alignment of each
* individual iovec segment, as they could end up with different
* I/Os due to the way bio_iov_iter_get_pages works, and we'd
* then overwrite an already written block.
*/
if (((iocb->ki_pos | count) & ip->i_mount->m_blockmask) ||
(xfs_is_always_cow_inode(ip) &&
(iov_iter_alignment(from) & ip->i_mount->m_blockmask)))
return xfs_file_dio_write_unaligned(ip, iocb, from);
return xfs_file_dio_write_aligned(ip, iocb, from);
if (xfs_is_zoned_inode(ip))
return xfs_file_dio_write_zoned(ip, iocb, from);
return xfs_file_dio_write_aligned(ip, iocb, from,
&xfs_direct_write_iomap_ops, &xfs_dio_write_ops, NULL);
}
static noinline ssize_t
@ -740,7 +858,7 @@ xfs_file_dax_write(
ret = xfs_ilock_iocb(iocb, iolock);
if (ret)
return ret;
ret = xfs_file_write_checks(iocb, from, &iolock);
ret = xfs_file_write_checks(iocb, from, &iolock, NULL);
if (ret)
goto out;
@ -784,7 +902,7 @@ xfs_file_buffered_write(
if (ret)
return ret;
ret = xfs_file_write_checks(iocb, from, &iolock);
ret = xfs_file_write_checks(iocb, from, &iolock, NULL);
if (ret)
goto out;
@ -831,6 +949,67 @@ xfs_file_buffered_write(
return ret;
}
STATIC ssize_t
xfs_file_buffered_write_zoned(
struct kiocb *iocb,
struct iov_iter *from)
{
struct xfs_inode *ip = XFS_I(iocb->ki_filp->f_mapping->host);
struct xfs_mount *mp = ip->i_mount;
unsigned int iolock = XFS_IOLOCK_EXCL;
bool cleared_space = false;
struct xfs_zone_alloc_ctx ac = { };
ssize_t ret;
ret = xfs_zoned_write_space_reserve(ip, iocb, from, XFS_ZR_GREEDY, &ac);
if (ret < 0)
return ret;
ret = xfs_ilock_iocb(iocb, iolock);
if (ret)
goto out_unreserve;
ret = xfs_file_write_checks(iocb, from, &iolock, &ac);
if (ret)
goto out_unlock;
/*
* Truncate the iter to the length that we were actually able to
* allocate blocks for. This needs to happen after
* xfs_file_write_checks, because that assigns ki_pos for O_APPEND
* writes.
*/
iov_iter_truncate(from,
XFS_FSB_TO_B(mp, ac.reserved_blocks) -
(iocb->ki_pos & mp->m_blockmask));
if (!iov_iter_count(from))
goto out_unlock;
retry:
trace_xfs_file_buffered_write(iocb, from);
ret = iomap_file_buffered_write(iocb, from,
&xfs_buffered_write_iomap_ops, &ac);
if (ret == -ENOSPC && !cleared_space) {
/*
* Kick off writeback to convert delalloc space and release the
* usually too pessimistic indirect block reservations.
*/
xfs_flush_inodes(mp);
cleared_space = true;
goto retry;
}
out_unlock:
xfs_iunlock(ip, iolock);
out_unreserve:
xfs_zoned_space_unreserve(ip, &ac);
if (ret > 0) {
XFS_STATS_ADD(mp, xs_write_bytes, ret);
ret = generic_write_sync(iocb, ret);
}
return ret;
}
STATIC ssize_t
xfs_file_write_iter(
struct kiocb *iocb,
@ -878,6 +1057,8 @@ xfs_file_write_iter(
return ret;
}
if (xfs_is_zoned_inode(ip))
return xfs_file_buffered_write_zoned(iocb, from);
return xfs_file_buffered_write(iocb, from);
}
@ -932,7 +1113,8 @@ static int
xfs_falloc_collapse_range(
struct file *file,
loff_t offset,
loff_t len)
loff_t len,
struct xfs_zone_alloc_ctx *ac)
{
struct inode *inode = file_inode(file);
loff_t new_size = i_size_read(inode) - len;
@ -948,7 +1130,7 @@ xfs_falloc_collapse_range(
if (offset + len >= i_size_read(inode))
return -EINVAL;
error = xfs_collapse_file_space(XFS_I(inode), offset, len);
error = xfs_collapse_file_space(XFS_I(inode), offset, len, ac);
if (error)
return error;
return xfs_falloc_setsize(file, new_size);
@ -1004,7 +1186,8 @@ xfs_falloc_zero_range(
struct file *file,
int mode,
loff_t offset,
loff_t len)
loff_t len,
struct xfs_zone_alloc_ctx *ac)
{
struct inode *inode = file_inode(file);
unsigned int blksize = i_blocksize(inode);
@ -1017,7 +1200,7 @@ xfs_falloc_zero_range(
if (error)
return error;
error = xfs_free_file_space(XFS_I(inode), offset, len);
error = xfs_free_file_space(XFS_I(inode), offset, len, ac);
if (error)
return error;
@ -1088,22 +1271,18 @@ xfs_falloc_allocate_range(
FALLOC_FL_INSERT_RANGE | FALLOC_FL_UNSHARE_RANGE)
STATIC long
xfs_file_fallocate(
__xfs_file_fallocate(
struct file *file,
int mode,
loff_t offset,
loff_t len)
loff_t len,
struct xfs_zone_alloc_ctx *ac)
{
struct inode *inode = file_inode(file);
struct xfs_inode *ip = XFS_I(inode);
long error;
uint iolock = XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL;
if (!S_ISREG(inode->i_mode))
return -EINVAL;
if (mode & ~XFS_FALLOC_FL_SUPPORTED)
return -EOPNOTSUPP;
xfs_ilock(ip, iolock);
error = xfs_break_layouts(inode, &iolock, BREAK_UNMAP);
if (error)
@ -1124,16 +1303,16 @@ xfs_file_fallocate(
switch (mode & FALLOC_FL_MODE_MASK) {
case FALLOC_FL_PUNCH_HOLE:
error = xfs_free_file_space(ip, offset, len);
error = xfs_free_file_space(ip, offset, len, ac);
break;
case FALLOC_FL_COLLAPSE_RANGE:
error = xfs_falloc_collapse_range(file, offset, len);
error = xfs_falloc_collapse_range(file, offset, len, ac);
break;
case FALLOC_FL_INSERT_RANGE:
error = xfs_falloc_insert_range(file, offset, len);
break;
case FALLOC_FL_ZERO_RANGE:
error = xfs_falloc_zero_range(file, mode, offset, len);
error = xfs_falloc_zero_range(file, mode, offset, len, ac);
break;
case FALLOC_FL_UNSHARE_RANGE:
error = xfs_falloc_unshare_range(file, mode, offset, len);
@ -1154,6 +1333,54 @@ xfs_file_fallocate(
return error;
}
static long
xfs_file_zoned_fallocate(
struct file *file,
int mode,
loff_t offset,
loff_t len)
{
struct xfs_zone_alloc_ctx ac = { };
struct xfs_inode *ip = XFS_I(file_inode(file));
int error;
error = xfs_zoned_space_reserve(ip, 2, XFS_ZR_RESERVED, &ac);
if (error)
return error;
error = __xfs_file_fallocate(file, mode, offset, len, &ac);
xfs_zoned_space_unreserve(ip, &ac);
return error;
}
static long
xfs_file_fallocate(
struct file *file,
int mode,
loff_t offset,
loff_t len)
{
struct inode *inode = file_inode(file);
if (!S_ISREG(inode->i_mode))
return -EINVAL;
if (mode & ~XFS_FALLOC_FL_SUPPORTED)
return -EOPNOTSUPP;
/*
* For zoned file systems, zeroing the first and last block of a hole
* punch requires allocating a new block to rewrite the remaining data
* and new zeroes out of place. Get a reservations for those before
* taking the iolock. Dip into the reserved pool because we are
* expected to be able to punch a hole even on a completely full
* file system.
*/
if (xfs_is_zoned_inode(XFS_I(inode)) &&
(mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE |
FALLOC_FL_COLLAPSE_RANGE)))
return xfs_file_zoned_fallocate(file, mode, offset, len);
return __xfs_file_fallocate(file, mode, offset, len, NULL);
}
STATIC int
xfs_file_fadvise(
struct file *file,
@ -1347,15 +1574,22 @@ xfs_file_release(
* blocks. This avoids open/read/close workloads from removing EOF
* blocks that other writers depend upon to reduce fragmentation.
*
* Inodes on the zoned RT device never have preallocations, so skip
* taking the locks below.
*/
if (!inode->i_nlink ||
!(file->f_mode & FMODE_WRITE) ||
(ip->i_diflags & XFS_DIFLAG_APPEND) ||
xfs_is_zoned_inode(ip))
return 0;
/*
* If we can't get the iolock just skip truncating the blocks past EOF
* because we could deadlock with the mmap_lock otherwise. We'll get
* another chance to drop them once the last reference to the inode is
* dropped, so we'll never leak blocks permanently.
*/
if (inode->i_nlink &&
(file->f_mode & FMODE_WRITE) &&
!(ip->i_diflags & XFS_DIFLAG_APPEND) &&
!xfs_iflags_test(ip, XFS_EOFBLOCKS_RELEASED) &&
if (!xfs_iflags_test(ip, XFS_EOFBLOCKS_RELEASED) &&
xfs_ilock_nowait(ip, XFS_IOLOCK_EXCL)) {
if (xfs_can_free_eofblocks(ip) &&
!xfs_iflags_test_and_set(ip, XFS_EOFBLOCKS_RELEASED))
@ -1469,9 +1703,10 @@ xfs_dax_read_fault(
* i_lock (XFS - extent map serialisation)
*/
static vm_fault_t
xfs_write_fault(
__xfs_write_fault(
struct vm_fault *vmf,
unsigned int order)
unsigned int order,
struct xfs_zone_alloc_ctx *ac)
{
struct inode *inode = file_inode(vmf->vma->vm_file);
struct xfs_inode *ip = XFS_I(inode);
@ -1498,13 +1733,50 @@ xfs_write_fault(
if (IS_DAX(inode))
ret = xfs_dax_fault_locked(vmf, order, true);
else
ret = iomap_page_mkwrite(vmf, &xfs_buffered_write_iomap_ops);
ret = iomap_page_mkwrite(vmf, &xfs_buffered_write_iomap_ops,
ac);
xfs_iunlock(ip, lock_mode);
sb_end_pagefault(inode->i_sb);
return ret;
}
static vm_fault_t
xfs_write_fault_zoned(
struct vm_fault *vmf,
unsigned int order)
{
struct xfs_inode *ip = XFS_I(file_inode(vmf->vma->vm_file));
unsigned int len = folio_size(page_folio(vmf->page));
struct xfs_zone_alloc_ctx ac = { };
int error;
vm_fault_t ret;
/*
* This could over-allocate as it doesn't check for truncation.
*
* But as the overallocation is limited to less than a folio and will be
* release instantly that's just fine.
*/
error = xfs_zoned_space_reserve(ip, XFS_B_TO_FSB(ip->i_mount, len), 0,
&ac);
if (error < 0)
return vmf_fs_error(error);
ret = __xfs_write_fault(vmf, order, &ac);
xfs_zoned_space_unreserve(ip, &ac);
return ret;
}
static vm_fault_t
xfs_write_fault(
struct vm_fault *vmf,
unsigned int order)
{
if (xfs_is_zoned_inode(XFS_I(file_inode(vmf->vma->vm_file))))
return xfs_write_fault_zoned(vmf, order);
return __xfs_write_fault(vmf, order, NULL);
}
static inline bool
xfs_is_write_fault(
struct vm_fault *vmf)
@ -1613,7 +1885,8 @@ const struct file_operations xfs_file_operations = {
.fadvise = xfs_file_fadvise,
.remap_file_range = xfs_file_remap_range,
.fop_flags = FOP_MMAP_SYNC | FOP_BUFFER_RASYNC |
FOP_BUFFER_WASYNC | FOP_DIO_PARALLEL_WRITE,
FOP_BUFFER_WASYNC | FOP_DIO_PARALLEL_WRITE |
FOP_DONTCACHE,
};
const struct file_operations xfs_dir_file_operations = {

86
fs/xfs/xfs_fsmap.c
View File

@ -879,17 +879,39 @@ xfs_getfsmap_rtdev_rmapbt(
struct xfs_mount *mp = tp->t_mountp;
struct xfs_rtgroup *rtg = NULL;
struct xfs_btree_cur *bt_cur = NULL;
xfs_daddr_t rtstart_daddr;
xfs_rtblock_t start_rtb;
xfs_rtblock_t end_rtb;
xfs_rgnumber_t start_rg, end_rg;
uint64_t eofs;
int error = 0;
eofs = XFS_FSB_TO_BB(mp, mp->m_sb.sb_rblocks);
eofs = XFS_FSB_TO_BB(mp, mp->m_sb.sb_rtstart + mp->m_sb.sb_rblocks);
if (keys[0].fmr_physical >= eofs)
return 0;
start_rtb = xfs_daddr_to_rtb(mp, keys[0].fmr_physical);
end_rtb = xfs_daddr_to_rtb(mp, min(eofs - 1, keys[1].fmr_physical));
rtstart_daddr = XFS_FSB_TO_BB(mp, mp->m_sb.sb_rtstart);
if (keys[0].fmr_physical < rtstart_daddr) {
struct xfs_fsmap_irec frec = {
.owner = XFS_RMAP_OWN_FS,
.len_daddr = rtstart_daddr,
};
/* Adjust the low key if we are continuing from where we left off. */
if (keys[0].fmr_length > 0) {
info->low_daddr = keys[0].fmr_physical + keys[0].fmr_length;
return 0;
}
/* Fabricate an rmap entry for space occupied by the data dev */
error = xfs_getfsmap_helper(tp, info, &frec);
if (error)
return error;
}
start_rtb = xfs_daddr_to_rtb(mp, rtstart_daddr + keys[0].fmr_physical);
end_rtb = xfs_daddr_to_rtb(mp, rtstart_daddr +
min(eofs - 1, keys[1].fmr_physical));
info->missing_owner = XFS_FMR_OWN_FREE;
@ -1004,22 +1026,40 @@ xfs_getfsmap_rtdev_rmapbt(
}
#endif /* CONFIG_XFS_RT */
static uint32_t
xfs_getfsmap_device(
struct xfs_mount *mp,
enum xfs_device dev)
{
if (mp->m_sb.sb_rtstart)
return dev;
switch (dev) {
case XFS_DEV_DATA:
return new_encode_dev(mp->m_ddev_targp->bt_dev);
case XFS_DEV_LOG:
return new_encode_dev(mp->m_logdev_targp->bt_dev);
case XFS_DEV_RT:
if (!mp->m_rtdev_targp)
break;
return new_encode_dev(mp->m_rtdev_targp->bt_dev);
}
return -1;
}
/* Do we recognize the device? */
STATIC bool
xfs_getfsmap_is_valid_device(
struct xfs_mount *mp,
struct xfs_fsmap *fm)
{
if (fm->fmr_device == 0 || fm->fmr_device == UINT_MAX ||
fm->fmr_device == new_encode_dev(mp->m_ddev_targp->bt_dev))
return true;
if (mp->m_logdev_targp &&
fm->fmr_device == new_encode_dev(mp->m_logdev_targp->bt_dev))
return true;
if (mp->m_rtdev_targp &&
fm->fmr_device == new_encode_dev(mp->m_rtdev_targp->bt_dev))
return true;
return false;
return fm->fmr_device == 0 ||
fm->fmr_device == UINT_MAX ||
fm->fmr_device == xfs_getfsmap_device(mp, XFS_DEV_DATA) ||
fm->fmr_device == xfs_getfsmap_device(mp, XFS_DEV_LOG) ||
(mp->m_rtdev_targp &&
fm->fmr_device == xfs_getfsmap_device(mp, XFS_DEV_RT));
}
/* Ensure that the low key is less than the high key. */
@ -1126,7 +1166,7 @@ xfs_getfsmap(
/* Set up our device handlers. */
memset(handlers, 0, sizeof(handlers));
handlers[0].nr_sectors = XFS_FSB_TO_BB(mp, mp->m_sb.sb_dblocks);
handlers[0].dev = new_encode_dev(mp->m_ddev_targp->bt_dev);
handlers[0].dev = xfs_getfsmap_device(mp, XFS_DEV_DATA);
if (use_rmap)
handlers[0].fn = xfs_getfsmap_datadev_rmapbt;
else
@ -1134,13 +1174,17 @@ xfs_getfsmap(
if (mp->m_logdev_targp != mp->m_ddev_targp) {
handlers[1].nr_sectors = XFS_FSB_TO_BB(mp,
mp->m_sb.sb_logblocks);
handlers[1].dev = new_encode_dev(mp->m_logdev_targp->bt_dev);
handlers[1].dev = xfs_getfsmap_device(mp, XFS_DEV_LOG);
handlers[1].fn = xfs_getfsmap_logdev;
}
#ifdef CONFIG_XFS_RT
if (mp->m_rtdev_targp) {
/*
* For zoned file systems there is no rtbitmap, so only support fsmap
* if the callers is privileged enough to use the full rmap version.
*/
if (mp->m_rtdev_targp && (use_rmap || !xfs_has_zoned(mp))) {
handlers[2].nr_sectors = XFS_FSB_TO_BB(mp, mp->m_sb.sb_rblocks);
handlers[2].dev = new_encode_dev(mp->m_rtdev_targp->bt_dev);
handlers[2].dev = xfs_getfsmap_device(mp, XFS_DEV_RT);
if (use_rmap)
handlers[2].fn = xfs_getfsmap_rtdev_rmapbt;
else
@ -1230,7 +1274,13 @@ xfs_getfsmap(
if (tp)
xfs_trans_cancel(tp);
head->fmh_oflags = FMH_OF_DEV_T;
/*
* For internal RT device we need to report different synthetic devices
* for a single physical device, and thus can't report the actual dev_t.
*/
if (!mp->m_sb.sb_rtstart)
head->fmh_oflags = FMH_OF_DEV_T;
return error;
}

65
fs/xfs/xfs_fsops.c
View File

@ -24,6 +24,7 @@
#include "xfs_rtalloc.h"
#include "xfs_rtrmap_btree.h"
#include "xfs_rtrefcount_btree.h"
#include "xfs_metafile.h"
/*
* Write new AG headers to disk. Non-transactional, but need to be
@ -300,24 +301,30 @@ xfs_growfs_data(
struct xfs_mount *mp,
struct xfs_growfs_data *in)
{
int error = 0;
int error;
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
if (!mutex_trylock(&mp->m_growlock))
return -EWOULDBLOCK;
/* we can't grow the data section when an internal RT section exists */
if (in->newblocks != mp->m_sb.sb_dblocks && mp->m_sb.sb_rtstart) {
error = -EINVAL;
goto out_unlock;
}
/* update imaxpct separately to the physical grow of the filesystem */
if (in->imaxpct != mp->m_sb.sb_imax_pct) {
error = xfs_growfs_imaxpct(mp, in->imaxpct);
if (error)
goto out_error;
goto out_unlock;
}
if (in->newblocks != mp->m_sb.sb_dblocks) {
error = xfs_growfs_data_private(mp, in);
if (error)
goto out_error;
goto out_unlock;
}
/* Post growfs calculations needed to reflect new state in operations */
@ -331,13 +338,12 @@ xfs_growfs_data(
/* Update secondary superblocks now the physical grow has completed */
error = xfs_update_secondary_sbs(mp);
out_error:
/*
* Increment the generation unconditionally, the error could be from
* updating the secondary superblocks, in which case the new size
* is live already.
* Increment the generation unconditionally, after trying to update the
* secondary superblocks, as the new size is live already at this point.
*/
mp->m_generation++;
out_unlock:
mutex_unlock(&mp->m_growlock);
return error;
}
@ -366,6 +372,7 @@ xfs_growfs_log(
int
xfs_reserve_blocks(
struct xfs_mount *mp,
enum xfs_free_counter ctr,
uint64_t request)
{
int64_t lcounter, delta;
@ -373,6 +380,8 @@ xfs_reserve_blocks(
int64_t free;
int error = 0;
ASSERT(ctr < XC_FREE_NR);
/*
* With per-cpu counters, this becomes an interesting problem. we need
* to work out if we are freeing or allocation blocks first, then we can
@ -391,16 +400,16 @@ xfs_reserve_blocks(
* counters directly since we shouldn't have any problems unreserving
* space.
*/
if (mp->m_resblks > request) {
lcounter = mp->m_resblks_avail - request;
if (mp->m_free[ctr].res_total > request) {
lcounter = mp->m_free[ctr].res_avail - request;
if (lcounter > 0) { /* release unused blocks */
fdblks_delta = lcounter;
mp->m_resblks_avail -= lcounter;
mp->m_free[ctr].res_avail -= lcounter;
}
mp->m_resblks = request;
mp->m_free[ctr].res_total = request;
if (fdblks_delta) {
spin_unlock(&mp->m_sb_lock);
xfs_add_fdblocks(mp, fdblks_delta);
xfs_add_freecounter(mp, ctr, fdblks_delta);
spin_lock(&mp->m_sb_lock);
}
@ -409,7 +418,7 @@ xfs_reserve_blocks(
/*
* If the request is larger than the current reservation, reserve the
* blocks before we update the reserve counters. Sample m_fdblocks and
* blocks before we update the reserve counters. Sample m_free and
* perform a partial reservation if the request exceeds free space.
*
* The code below estimates how many blocks it can request from
@ -419,10 +428,10 @@ xfs_reserve_blocks(
* space to fill it because mod_fdblocks will refill an undersized
* reserve when it can.
*/
free = percpu_counter_sum(&mp->m_fdblocks) -
xfs_fdblocks_unavailable(mp);
delta = request - mp->m_resblks;
mp->m_resblks = request;
free = xfs_sum_freecounter_raw(mp, ctr) -
xfs_freecounter_unavailable(mp, ctr);
delta = request - mp->m_free[ctr].res_total;
mp->m_free[ctr].res_total = request;
if (delta > 0 && free > 0) {
/*
* We'll either succeed in getting space from the free block
@ -436,9 +445,9 @@ xfs_reserve_blocks(
*/
fdblks_delta = min(free, delta);
spin_unlock(&mp->m_sb_lock);
error = xfs_dec_fdblocks(mp, fdblks_delta, 0);
error = xfs_dec_freecounter(mp, ctr, fdblks_delta, 0);
if (!error)
xfs_add_fdblocks(mp, fdblks_delta);
xfs_add_freecounter(mp, ctr, fdblks_delta);
spin_lock(&mp->m_sb_lock);
}
out:
@ -558,15 +567,13 @@ xfs_fs_reserve_ag_blocks(
return error;
}
if (xfs_has_realtime(mp)) {
err2 = xfs_rt_resv_init(mp);
if (err2 && err2 != -ENOSPC) {
xfs_warn(mp,
"Error %d reserving realtime metadata reserve pool.", err2);
xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE);
}
err2 = xfs_metafile_resv_init(mp);
if (err2 && err2 != -ENOSPC) {
xfs_warn(mp,
"Error %d reserving realtime metadata reserve pool.", err2);
xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE);
if (err2 && !error)
if (!error)
error = err2;
}
@ -582,9 +589,7 @@ xfs_fs_unreserve_ag_blocks(
{
struct xfs_perag *pag = NULL;
if (xfs_has_realtime(mp))
xfs_rt_resv_free(mp);
xfs_metafile_resv_free(mp);
while ((pag = xfs_perag_next(mp, pag)))
xfs_ag_resv_free(pag);
}

3
fs/xfs/xfs_fsops.h
View File

@ -8,7 +8,8 @@
int xfs_growfs_data(struct xfs_mount *mp, struct xfs_growfs_data *in);
int xfs_growfs_log(struct xfs_mount *mp, struct xfs_growfs_log *in);
int xfs_reserve_blocks(struct xfs_mount *mp, uint64_t request);
int xfs_reserve_blocks(struct xfs_mount *mp, enum xfs_free_counter cnt,
uint64_t request);
int xfs_fs_goingdown(struct xfs_mount *mp, uint32_t inflags);
int xfs_fs_reserve_ag_blocks(struct xfs_mount *mp);

6
fs/xfs/xfs_icache.c
View File

@ -2073,10 +2073,10 @@ xfs_inodegc_want_queue_rt_file(
{
struct xfs_mount *mp = ip->i_mount;
if (!XFS_IS_REALTIME_INODE(ip))
if (!XFS_IS_REALTIME_INODE(ip) || xfs_has_zoned(mp))
return false;
if (__percpu_counter_compare(&mp->m_frextents,
if (xfs_compare_freecounter(mp, XC_FREE_RTEXTENTS,
mp->m_low_rtexts[XFS_LOWSP_5_PCNT],
XFS_FDBLOCKS_BATCH) < 0)
return true;
@ -2104,7 +2104,7 @@ xfs_inodegc_want_queue_work(
if (items > mp->m_ino_geo.inodes_per_cluster)
return true;
if (__percpu_counter_compare(&mp->m_fdblocks,
if (xfs_compare_freecounter(mp, XC_FREE_BLOCKS,
mp->m_low_space[XFS_LOWSP_5_PCNT],
XFS_FDBLOCKS_BATCH) < 0)
return true;

3
fs/xfs/xfs_inode.c
View File

@ -3074,5 +3074,6 @@ bool
xfs_is_always_cow_inode(
const struct xfs_inode *ip)
{
return ip->i_mount->m_always_cow && xfs_has_reflink(ip->i_mount);
return xfs_is_zoned_inode(ip) ||
(ip->i_mount->m_always_cow && xfs_has_reflink(ip->i_mount));
}

28
fs/xfs/xfs_inode.h
View File

@ -25,19 +25,9 @@ struct xfs_dquot;
typedef struct xfs_inode {
/* Inode linking and identification information. */
struct xfs_mount *i_mount; /* fs mount struct ptr */
union {
struct {
struct xfs_dquot *i_udquot; /* user dquot */
struct xfs_dquot *i_gdquot; /* group dquot */
struct xfs_dquot *i_pdquot; /* project dquot */
};
/*
* Space that has been set aside to accomodate expansions of a
* metadata btree rooted in this file.
*/
uint64_t i_meta_resv_asked;
};
struct xfs_dquot *i_udquot; /* user dquot */
struct xfs_dquot *i_gdquot; /* group dquot */
struct xfs_dquot *i_pdquot; /* project dquot */
/* Inode location stuff */
xfs_ino_t i_ino; /* inode number (agno/agino)*/
@ -69,8 +59,13 @@ typedef struct xfs_inode {
xfs_rfsblock_t i_nblocks; /* # of direct & btree blocks */
prid_t i_projid; /* owner's project id */
xfs_extlen_t i_extsize; /* basic/minimum extent size */
/* cowextsize is only used for v3 inodes, flushiter for v1/2 */
/*
* i_used_blocks is used for zoned rtrmap inodes,
* i_cowextsize is used for other v3 inodes,
* i_flushiter for v1/2 inodes
*/
union {
uint32_t i_used_blocks; /* used blocks in RTG */
xfs_extlen_t i_cowextsize; /* basic cow extent size */
uint16_t i_flushiter; /* incremented on flush */
};
@ -309,6 +304,11 @@ static inline bool xfs_is_internal_inode(const struct xfs_inode *ip)
xfs_is_quota_inode(&mp->m_sb, ip->i_ino);
}
static inline bool xfs_is_zoned_inode(const struct xfs_inode *ip)
{
return xfs_has_zoned(ip->i_mount) && XFS_IS_REALTIME_INODE(ip);
}
bool xfs_is_always_cow_inode(const struct xfs_inode *ip);
static inline bool xfs_is_cow_inode(const struct xfs_inode *ip)

1
fs/xfs/xfs_inode_item.c
View File

@ -596,6 +596,7 @@ xfs_inode_to_log_dinode(
to->di_changecount = inode_peek_iversion(inode);
to->di_crtime = xfs_inode_to_log_dinode_ts(ip, ip->i_crtime);
to->di_flags2 = ip->i_diflags2;
/* also covers the di_used_blocks union arm: */
to->di_cowextsize = ip->i_cowextsize;
to->di_ino = ip->i_ino;
to->di_lsn = lsn;

1
fs/xfs/xfs_inode_item_recover.c
View File

@ -203,6 +203,7 @@ xfs_log_dinode_to_disk(
to->di_crtime = xfs_log_dinode_to_disk_ts(from,
from->di_crtime);
to->di_flags2 = cpu_to_be64(from->di_flags2);
/* also covers the di_used_blocks union arm: */
to->di_cowextsize = cpu_to_be32(from->di_cowextsize);
to->di_ino = cpu_to_be64(from->di_ino);
to->di_lsn = cpu_to_be64(lsn);

12
fs/xfs/xfs_ioctl.c
View File

@ -1131,15 +1131,15 @@ xfs_ioctl_getset_resblocks(
error = mnt_want_write_file(filp);
if (error)
return error;
error = xfs_reserve_blocks(mp, fsop.resblks);
error = xfs_reserve_blocks(mp, XC_FREE_BLOCKS, fsop.resblks);
mnt_drop_write_file(filp);
if (error)
return error;
}
spin_lock(&mp->m_sb_lock);
fsop.resblks = mp->m_resblks;
fsop.resblks_avail = mp->m_resblks_avail;
fsop.resblks = mp->m_free[XC_FREE_BLOCKS].res_total;
fsop.resblks_avail = mp->m_free[XC_FREE_BLOCKS].res_avail;
spin_unlock(&mp->m_sb_lock);
if (copy_to_user(arg, &fsop, sizeof(fsop)))
@ -1155,9 +1155,9 @@ xfs_ioctl_fs_counts(
struct xfs_fsop_counts out = {
.allocino = percpu_counter_read_positive(&mp->m_icount),
.freeino = percpu_counter_read_positive(&mp->m_ifree),
.freedata = percpu_counter_read_positive(&mp->m_fdblocks) -
xfs_fdblocks_unavailable(mp),
.freertx = percpu_counter_read_positive(&mp->m_frextents),
.freedata = xfs_estimate_freecounter(mp, XC_FREE_BLOCKS) -
xfs_freecounter_unavailable(mp, XC_FREE_BLOCKS),
.freertx = xfs_estimate_freecounter(mp, XC_FREE_RTEXTENTS),
};
if (copy_to_user(uarg, &out, sizeof(out)))

528
fs/xfs/xfs_iomap.c
View File

@ -30,6 +30,8 @@
#include "xfs_reflink.h"
#include "xfs_health.h"
#include "xfs_rtbitmap.h"
#include "xfs_icache.h"
#include "xfs_zone_alloc.h"
#define XFS_ALLOC_ALIGN(mp, off) \
(((off) >> mp->m_allocsize_log) << mp->m_allocsize_log)
@ -431,13 +433,14 @@ xfs_quota_calc_throttle(
static int64_t
xfs_iomap_freesp(
struct percpu_counter *counter,
struct xfs_mount *mp,
unsigned int idx,
uint64_t low_space[XFS_LOWSP_MAX],
int *shift)
{
int64_t freesp;
freesp = percpu_counter_read_positive(counter);
freesp = xfs_estimate_freecounter(mp, idx);
if (freesp < low_space[XFS_LOWSP_5_PCNT]) {
*shift = 2;
if (freesp < low_space[XFS_LOWSP_4_PCNT])
@ -536,10 +539,10 @@ xfs_iomap_prealloc_size(
if (unlikely(XFS_IS_REALTIME_INODE(ip)))
freesp = xfs_rtbxlen_to_blen(mp,
xfs_iomap_freesp(&mp->m_frextents,
xfs_iomap_freesp(mp, XC_FREE_RTEXTENTS,
mp->m_low_rtexts, &shift));
else
freesp = xfs_iomap_freesp(&mp->m_fdblocks, mp->m_low_space,
freesp = xfs_iomap_freesp(mp, XC_FREE_BLOCKS, mp->m_low_space,
&shift);
/*
@ -962,6 +965,59 @@ const struct iomap_ops xfs_direct_write_iomap_ops = {
.iomap_begin = xfs_direct_write_iomap_begin,
};
#ifdef CONFIG_XFS_RT
/*
* This is really simple. The space has already been reserved before taking the
* IOLOCK, the actual block allocation is done just before submitting the bio
* and only recorded in the extent map on I/O completion.
*/
static int
xfs_zoned_direct_write_iomap_begin(
struct inode *inode,
loff_t offset,
loff_t length,
unsigned flags,
struct iomap *iomap,
struct iomap *srcmap)
{
struct xfs_inode *ip = XFS_I(inode);
int error;
ASSERT(!(flags & IOMAP_OVERWRITE_ONLY));
/*
* Needs to be pushed down into the allocator so that only writes into
* a single zone can be supported.
*/
if (flags & IOMAP_NOWAIT)
return -EAGAIN;
/*
* Ensure the extent list is in memory in so that we don't have to do
* read it from the I/O completion handler.
*/
if (xfs_need_iread_extents(&ip->i_df)) {
xfs_ilock(ip, XFS_ILOCK_EXCL);
error = xfs_iread_extents(NULL, ip, XFS_DATA_FORK);
xfs_iunlock(ip, XFS_ILOCK_EXCL);
if (error)
return error;
}
iomap->type = IOMAP_MAPPED;
iomap->flags = IOMAP_F_DIRTY;
iomap->bdev = ip->i_mount->m_rtdev_targp->bt_bdev;
iomap->offset = offset;
iomap->length = length;
iomap->flags = IOMAP_F_ANON_WRITE;
return 0;
}
const struct iomap_ops xfs_zoned_direct_write_iomap_ops = {
.iomap_begin = xfs_zoned_direct_write_iomap_begin,
};
#endif /* CONFIG_XFS_RT */
static int
xfs_dax_write_iomap_end(
struct inode *inode,
@ -987,6 +1043,455 @@ const struct iomap_ops xfs_dax_write_iomap_ops = {
.iomap_end = xfs_dax_write_iomap_end,
};
/*
* Convert a hole to a delayed allocation.
*/
static void
xfs_bmap_add_extent_hole_delay(
struct xfs_inode *ip, /* incore inode pointer */
int whichfork,
struct xfs_iext_cursor *icur,
struct xfs_bmbt_irec *new) /* new data to add to file extents */
{
struct xfs_ifork *ifp; /* inode fork pointer */
xfs_bmbt_irec_t left; /* left neighbor extent entry */
xfs_filblks_t newlen=0; /* new indirect size */
xfs_filblks_t oldlen=0; /* old indirect size */
xfs_bmbt_irec_t right; /* right neighbor extent entry */
uint32_t state = xfs_bmap_fork_to_state(whichfork);
xfs_filblks_t temp; /* temp for indirect calculations */
ifp = xfs_ifork_ptr(ip, whichfork);
ASSERT(isnullstartblock(new->br_startblock));
/*
* Check and set flags if this segment has a left neighbor
*/
if (xfs_iext_peek_prev_extent(ifp, icur, &left)) {
state |= BMAP_LEFT_VALID;
if (isnullstartblock(left.br_startblock))
state |= BMAP_LEFT_DELAY;
}
/*
* Check and set flags if the current (right) segment exists.
* If it doesn't exist, we're converting the hole at end-of-file.
*/
if (xfs_iext_get_extent(ifp, icur, &right)) {
state |= BMAP_RIGHT_VALID;
if (isnullstartblock(right.br_startblock))
state |= BMAP_RIGHT_DELAY;
}
/*
* Set contiguity flags on the left and right neighbors.
* Don't let extents get too large, even if the pieces are contiguous.
*/
if ((state & BMAP_LEFT_VALID) && (state & BMAP_LEFT_DELAY) &&
left.br_startoff + left.br_blockcount == new->br_startoff &&
left.br_blockcount + new->br_blockcount <= XFS_MAX_BMBT_EXTLEN)
state |= BMAP_LEFT_CONTIG;
if ((state & BMAP_RIGHT_VALID) && (state & BMAP_RIGHT_DELAY) &&
new->br_startoff + new->br_blockcount == right.br_startoff &&
new->br_blockcount + right.br_blockcount <= XFS_MAX_BMBT_EXTLEN &&
(!(state & BMAP_LEFT_CONTIG) ||
(left.br_blockcount + new->br_blockcount +
right.br_blockcount <= XFS_MAX_BMBT_EXTLEN)))
state |= BMAP_RIGHT_CONTIG;
/*
* Switch out based on the contiguity flags.
*/
switch (state & (BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG)) {
case BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG:
/*
* New allocation is contiguous with delayed allocations
* on the left and on the right.
* Merge all three into a single extent record.
*/
temp = left.br_blockcount + new->br_blockcount +
right.br_blockcount;
oldlen = startblockval(left.br_startblock) +
startblockval(new->br_startblock) +
startblockval(right.br_startblock);
newlen = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(ip, temp),
oldlen);
left.br_startblock = nullstartblock(newlen);
left.br_blockcount = temp;
xfs_iext_remove(ip, icur, state);
xfs_iext_prev(ifp, icur);
xfs_iext_update_extent(ip, state, icur, &left);
break;
case BMAP_LEFT_CONTIG:
/*
* New allocation is contiguous with a delayed allocation
* on the left.
* Merge the new allocation with the left neighbor.
*/
temp = left.br_blockcount + new->br_blockcount;
oldlen = startblockval(left.br_startblock) +
startblockval(new->br_startblock);
newlen = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(ip, temp),
oldlen);
left.br_blockcount = temp;
left.br_startblock = nullstartblock(newlen);
xfs_iext_prev(ifp, icur);
xfs_iext_update_extent(ip, state, icur, &left);
break;
case BMAP_RIGHT_CONTIG:
/*
* New allocation is contiguous with a delayed allocation
* on the right.
* Merge the new allocation with the right neighbor.
*/
temp = new->br_blockcount + right.br_blockcount;
oldlen = startblockval(new->br_startblock) +
startblockval(right.br_startblock);
newlen = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(ip, temp),
oldlen);
right.br_startoff = new->br_startoff;
right.br_startblock = nullstartblock(newlen);
right.br_blockcount = temp;
xfs_iext_update_extent(ip, state, icur, &right);
break;
case 0:
/*
* New allocation is not contiguous with another
* delayed allocation.
* Insert a new entry.
*/
oldlen = newlen = 0;
xfs_iext_insert(ip, icur, new, state);
break;
}
if (oldlen != newlen) {
ASSERT(oldlen > newlen);
xfs_add_fdblocks(ip->i_mount, oldlen - newlen);
/*
* Nothing to do for disk quota accounting here.
*/
xfs_mod_delalloc(ip, 0, (int64_t)newlen - oldlen);
}
}
/*
* Add a delayed allocation extent to an inode. Blocks are reserved from the
* global pool and the extent inserted into the inode in-core extent tree.
*
* On entry, got refers to the first extent beyond the offset of the extent to
* allocate or eof is specified if no such extent exists. On return, got refers
* to the extent record that was inserted to the inode fork.
*
* Note that the allocated extent may have been merged with contiguous extents
* during insertion into the inode fork. Thus, got does not reflect the current
* state of the inode fork on return. If necessary, the caller can use lastx to
* look up the updated record in the inode fork.
*/
static int
xfs_bmapi_reserve_delalloc(
struct xfs_inode *ip,
int whichfork,
xfs_fileoff_t off,
xfs_filblks_t len,
xfs_filblks_t prealloc,
struct xfs_bmbt_irec *got,
struct xfs_iext_cursor *icur,
int eof)
{
struct xfs_mount *mp = ip->i_mount;
struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork);
xfs_extlen_t alen;
xfs_extlen_t indlen;
uint64_t fdblocks;
int error;
xfs_fileoff_t aoff;
bool use_cowextszhint =
whichfork == XFS_COW_FORK && !prealloc;
retry:
/*
* Cap the alloc length. Keep track of prealloc so we know whether to
* tag the inode before we return.
*/
aoff = off;
alen = XFS_FILBLKS_MIN(len + prealloc, XFS_MAX_BMBT_EXTLEN);
if (!eof)
alen = XFS_FILBLKS_MIN(alen, got->br_startoff - aoff);
if (prealloc && alen >= len)
prealloc = alen - len;
/*
* If we're targetting the COW fork but aren't creating a speculative
* posteof preallocation, try to expand the reservation to align with
* the COW extent size hint if there's sufficient free space.
*
* Unlike the data fork, the CoW cancellation functions will free all
* the reservations at inactivation, so we don't require that every
* delalloc reservation have a dirty pagecache.
*/
if (use_cowextszhint) {
struct xfs_bmbt_irec prev;
xfs_extlen_t extsz = xfs_get_cowextsz_hint(ip);
if (!xfs_iext_peek_prev_extent(ifp, icur, &prev))
prev.br_startoff = NULLFILEOFF;
error = xfs_bmap_extsize_align(mp, got, &prev, extsz, 0, eof,
1, 0, &aoff, &alen);
ASSERT(!error);
}
/*
* Make a transaction-less quota reservation for delayed allocation
* blocks. This number gets adjusted later. We return if we haven't
* allocated blocks already inside this loop.
*/
error = xfs_quota_reserve_blkres(ip, alen);
if (error)
goto out;
/*
* Split changing sb for alen and indlen since they could be coming
* from different places.
*/
indlen = (xfs_extlen_t)xfs_bmap_worst_indlen(ip, alen);
ASSERT(indlen > 0);
fdblocks = indlen;
if (XFS_IS_REALTIME_INODE(ip)) {
ASSERT(!xfs_is_zoned_inode(ip));
error = xfs_dec_frextents(mp, xfs_blen_to_rtbxlen(mp, alen));
if (error)
goto out_unreserve_quota;
} else {
fdblocks += alen;
}
error = xfs_dec_fdblocks(mp, fdblocks, false);
if (error)
goto out_unreserve_frextents;
ip->i_delayed_blks += alen;
xfs_mod_delalloc(ip, alen, indlen);
got->br_startoff = aoff;
got->br_startblock = nullstartblock(indlen);
got->br_blockcount = alen;
got->br_state = XFS_EXT_NORM;
xfs_bmap_add_extent_hole_delay(ip, whichfork, icur, got);
/*
* Tag the inode if blocks were preallocated. Note that COW fork
* preallocation can occur at the start or end of the extent, even when
* prealloc == 0, so we must also check the aligned offset and length.
*/
if (whichfork == XFS_DATA_FORK && prealloc)
xfs_inode_set_eofblocks_tag(ip);
if (whichfork == XFS_COW_FORK && (prealloc || aoff < off || alen > len))
xfs_inode_set_cowblocks_tag(ip);
return 0;
out_unreserve_frextents:
if (XFS_IS_REALTIME_INODE(ip))
xfs_add_frextents(mp, xfs_blen_to_rtbxlen(mp, alen));
out_unreserve_quota:
if (XFS_IS_QUOTA_ON(mp))
xfs_quota_unreserve_blkres(ip, alen);
out:
if (error == -ENOSPC || error == -EDQUOT) {
trace_xfs_delalloc_enospc(ip, off, len);
if (prealloc || use_cowextszhint) {
/* retry without any preallocation */
use_cowextszhint = false;
prealloc = 0;
goto retry;
}
}
return error;
}
static int
xfs_zoned_buffered_write_iomap_begin(
struct inode *inode,
loff_t offset,
loff_t count,
unsigned flags,
struct iomap *iomap,
struct iomap *srcmap)
{
struct iomap_iter *iter =
container_of(iomap, struct iomap_iter, iomap);
struct xfs_zone_alloc_ctx *ac = iter->private;
struct xfs_inode *ip = XFS_I(inode);
struct xfs_mount *mp = ip->i_mount;
xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset);
xfs_fileoff_t end_fsb = xfs_iomap_end_fsb(mp, offset, count);
u16 iomap_flags = IOMAP_F_SHARED;
unsigned int lockmode = XFS_ILOCK_EXCL;
xfs_filblks_t count_fsb;
xfs_extlen_t indlen;
struct xfs_bmbt_irec got;
struct xfs_iext_cursor icur;
int error = 0;
ASSERT(!xfs_get_extsz_hint(ip));
ASSERT(!(flags & IOMAP_UNSHARE));
ASSERT(ac);
if (xfs_is_shutdown(mp))
return -EIO;
error = xfs_qm_dqattach(ip);
if (error)
return error;
error = xfs_ilock_for_iomap(ip, flags, &lockmode);
if (error)
return error;
if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(&ip->i_df)) ||
XFS_TEST_ERROR(false, mp, XFS_ERRTAG_BMAPIFORMAT)) {
xfs_bmap_mark_sick(ip, XFS_DATA_FORK);
error = -EFSCORRUPTED;
goto out_unlock;
}
XFS_STATS_INC(mp, xs_blk_mapw);
error = xfs_iread_extents(NULL, ip, XFS_DATA_FORK);
if (error)
goto out_unlock;
/*
* For zeroing operations check if there is any data to zero first.
*
* For regular writes we always need to allocate new blocks, but need to
* provide the source mapping when the range is unaligned to support
* read-modify-write of the whole block in the page cache.
*
* In either case we need to limit the reported range to the boundaries
* of the source map in the data fork.
*/
if (!IS_ALIGNED(offset, mp->m_sb.sb_blocksize) ||
!IS_ALIGNED(offset + count, mp->m_sb.sb_blocksize) ||
(flags & IOMAP_ZERO)) {
struct xfs_bmbt_irec smap;
struct xfs_iext_cursor scur;
if (!xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &scur,
&smap))
smap.br_startoff = end_fsb; /* fake hole until EOF */
if (smap.br_startoff > offset_fsb) {
/*
* We never need to allocate blocks for zeroing a hole.
*/
if (flags & IOMAP_ZERO) {
xfs_hole_to_iomap(ip, iomap, offset_fsb,
smap.br_startoff);
goto out_unlock;
}
end_fsb = min(end_fsb, smap.br_startoff);
} else {
end_fsb = min(end_fsb,
smap.br_startoff + smap.br_blockcount);
xfs_trim_extent(&smap, offset_fsb,
end_fsb - offset_fsb);
error = xfs_bmbt_to_iomap(ip, srcmap, &smap, flags, 0,
xfs_iomap_inode_sequence(ip, 0));
if (error)
goto out_unlock;
}
}
if (!ip->i_cowfp)
xfs_ifork_init_cow(ip);
if (!xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &got))
got.br_startoff = end_fsb;
if (got.br_startoff <= offset_fsb) {
trace_xfs_reflink_cow_found(ip, &got);
goto done;
}
/*
* Cap the maximum length to keep the chunks of work done here somewhat
* symmetric with the work writeback does.
*/
end_fsb = min(end_fsb, got.br_startoff);
count_fsb = min3(end_fsb - offset_fsb, XFS_MAX_BMBT_EXTLEN,
XFS_B_TO_FSB(mp, 1024 * PAGE_SIZE));
/*
* The block reservation is supposed to cover all blocks that the
* operation could possible write, but there is a nasty corner case
* where blocks could be stolen from underneath us:
*
* 1) while this thread iterates over a larger buffered write,
* 2) another thread is causing a write fault that calls into
* ->page_mkwrite in range this thread writes to, using up the
* delalloc reservation created by a previous call to this function.
* 3) another thread does direct I/O on the range that the write fault
* happened on, which causes writeback of the dirty data.
* 4) this then set the stale flag, which cuts the current iomap
* iteration short, causing the new call to ->iomap_begin that gets
* us here again, but now without a sufficient reservation.
*
* This is a very unusual I/O pattern, and nothing but generic/095 is
* known to hit it. There's not really much we can do here, so turn this
* into a short write.
*/
if (count_fsb > ac->reserved_blocks) {
xfs_warn_ratelimited(mp,
"Short write on ino 0x%llx comm %.20s due to three-way race with write fault and direct I/O",
ip->i_ino, current->comm);
count_fsb = ac->reserved_blocks;
if (!count_fsb) {
error = -EIO;
goto out_unlock;
}
}
error = xfs_quota_reserve_blkres(ip, count_fsb);
if (error)
goto out_unlock;
indlen = xfs_bmap_worst_indlen(ip, count_fsb);
error = xfs_dec_fdblocks(mp, indlen, false);
if (error)
goto out_unlock;
ip->i_delayed_blks += count_fsb;
xfs_mod_delalloc(ip, count_fsb, indlen);
got.br_startoff = offset_fsb;
got.br_startblock = nullstartblock(indlen);
got.br_blockcount = count_fsb;
got.br_state = XFS_EXT_NORM;
xfs_bmap_add_extent_hole_delay(ip, XFS_COW_FORK, &icur, &got);
ac->reserved_blocks -= count_fsb;
iomap_flags |= IOMAP_F_NEW;
trace_xfs_iomap_alloc(ip, offset, XFS_FSB_TO_B(mp, count_fsb),
XFS_COW_FORK, &got);
done:
error = xfs_bmbt_to_iomap(ip, iomap, &got, flags, iomap_flags,
xfs_iomap_inode_sequence(ip, IOMAP_F_SHARED));
out_unlock:
xfs_iunlock(ip, lockmode);
return error;
}
static int
xfs_buffered_write_iomap_begin(
struct inode *inode,
@ -1013,6 +1518,10 @@ xfs_buffered_write_iomap_begin(
if (xfs_is_shutdown(mp))
return -EIO;
if (xfs_is_zoned_inode(ip))
return xfs_zoned_buffered_write_iomap_begin(inode, offset,
count, flags, iomap, srcmap);
/* we can't use delayed allocations when using extent size hints */
if (xfs_get_extsz_hint(ip))
return xfs_direct_write_iomap_begin(inode, offset, count,
@ -1245,10 +1754,13 @@ xfs_buffered_write_delalloc_punch(
loff_t length,
struct iomap *iomap)
{
struct iomap_iter *iter =
container_of(iomap, struct iomap_iter, iomap);
xfs_bmap_punch_delalloc_range(XFS_I(inode),
(iomap->flags & IOMAP_F_SHARED) ?
XFS_COW_FORK : XFS_DATA_FORK,
offset, offset + length);
offset, offset + length, iter->private);
}
static int
@ -1485,6 +1997,7 @@ xfs_zero_range(
struct xfs_inode *ip,
loff_t pos,
loff_t len,
struct xfs_zone_alloc_ctx *ac,
bool *did_zero)
{
struct inode *inode = VFS_I(ip);
@ -1495,13 +2008,14 @@ xfs_zero_range(
return dax_zero_range(inode, pos, len, did_zero,
&xfs_dax_write_iomap_ops);
return iomap_zero_range(inode, pos, len, did_zero,
&xfs_buffered_write_iomap_ops);
&xfs_buffered_write_iomap_ops, ac);
}
int
xfs_truncate_page(
struct xfs_inode *ip,
loff_t pos,
struct xfs_zone_alloc_ctx *ac,
bool *did_zero)
{
struct inode *inode = VFS_I(ip);
@ -1510,5 +2024,5 @@ xfs_truncate_page(
return dax_truncate_page(inode, pos, did_zero,
&xfs_dax_write_iomap_ops);
return iomap_truncate_page(inode, pos, did_zero,
&xfs_buffered_write_iomap_ops);
&xfs_buffered_write_iomap_ops, ac);
}

7
fs/xfs/xfs_iomap.h
View File

@ -10,6 +10,7 @@
struct xfs_inode;
struct xfs_bmbt_irec;
struct xfs_zone_alloc_ctx;
int xfs_iomap_write_direct(struct xfs_inode *ip, xfs_fileoff_t offset_fsb,
xfs_fileoff_t count_fsb, unsigned int flags,
@ -24,8 +25,9 @@ int xfs_bmbt_to_iomap(struct xfs_inode *ip, struct iomap *iomap,
u16 iomap_flags, u64 sequence_cookie);
int xfs_zero_range(struct xfs_inode *ip, loff_t pos, loff_t len,
bool *did_zero);
int xfs_truncate_page(struct xfs_inode *ip, loff_t pos, bool *did_zero);
struct xfs_zone_alloc_ctx *ac, bool *did_zero);
int xfs_truncate_page(struct xfs_inode *ip, loff_t pos,
struct xfs_zone_alloc_ctx *ac, bool *did_zero);
static inline xfs_filblks_t
xfs_aligned_fsb_count(
@ -49,6 +51,7 @@ xfs_aligned_fsb_count(
extern const struct iomap_ops xfs_buffered_write_iomap_ops;
extern const struct iomap_ops xfs_direct_write_iomap_ops;
extern const struct iomap_ops xfs_zoned_direct_write_iomap_ops;
extern const struct iomap_ops xfs_read_iomap_ops;
extern const struct iomap_ops xfs_seek_iomap_ops;
extern const struct iomap_ops xfs_xattr_iomap_ops;

31
fs/xfs/xfs_iops.c
View File

@ -29,6 +29,7 @@
#include "xfs_xattr.h"
#include "xfs_file.h"
#include "xfs_bmap.h"
#include "xfs_zone_alloc.h"
#include <linux/posix_acl.h>
#include <linux/security.h>
@ -854,6 +855,7 @@ xfs_setattr_size(
uint lock_flags = 0;
uint resblks = 0;
bool did_zeroing = false;
struct xfs_zone_alloc_ctx ac = { };
xfs_assert_ilocked(ip, XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL);
ASSERT(S_ISREG(inode->i_mode));
@ -889,6 +891,28 @@ xfs_setattr_size(
*/
inode_dio_wait(inode);
/*
* Normally xfs_zoned_space_reserve is supposed to be called outside the
* IOLOCK. For truncate we can't do that since ->setattr is called with
* it already held by the VFS. So for now chicken out and try to
* allocate space under it.
*
* To avoid deadlocks this means we can't block waiting for space, which
* can lead to spurious -ENOSPC if there are no directly available
* blocks. We mitigate this a bit by allowing zeroing to dip into the
* reserved pool, but eventually the VFS calling convention needs to
* change.
*/
if (xfs_is_zoned_inode(ip)) {
error = xfs_zoned_space_reserve(ip, 1,
XFS_ZR_NOWAIT | XFS_ZR_RESERVED, &ac);
if (error) {
if (error == -EAGAIN)
return -ENOSPC;
return error;
}
}
/*
* File data changes must be complete before we start the transaction to
* modify the inode. This needs to be done before joining the inode to
@ -902,11 +926,14 @@ xfs_setattr_size(
if (newsize > oldsize) {
trace_xfs_zero_eof(ip, oldsize, newsize - oldsize);
error = xfs_zero_range(ip, oldsize, newsize - oldsize,
&did_zeroing);
&ac, &did_zeroing);
} else {
error = xfs_truncate_page(ip, newsize, &did_zeroing);
error = xfs_truncate_page(ip, newsize, &ac, &did_zeroing);
}
if (xfs_is_zoned_inode(ip))
xfs_zoned_space_unreserve(ip, &ac);
if (error)
return error;

4
fs/xfs/xfs_log.c
View File

@ -20,6 +20,7 @@
#include "xfs_sysfs.h"
#include "xfs_sb.h"
#include "xfs_health.h"
#include "xfs_zone_alloc.h"
struct kmem_cache *xfs_log_ticket_cache;
@ -3540,6 +3541,9 @@ xlog_force_shutdown(
spin_unlock(&log->l_icloglock);
wake_up_var(&log->l_opstate);
if (IS_ENABLED(CONFIG_XFS_RT) && xfs_has_zoned(log->l_mp))
xfs_zoned_wake_all(log->l_mp);
return log_error;
}

4
fs/xfs/xfs_message.c
View File

@ -173,6 +173,10 @@ xfs_warn_experimental(
.opstate = XFS_OPSTATE_WARNED_METADIR,
.name = "metadata directory tree",
},
[XFS_EXPERIMENTAL_ZONED] = {
.opstate = XFS_OPSTATE_WARNED_ZONED,
.name = "zoned RT device",
},
};
ASSERT(feat >= 0 && feat < XFS_EXPERIMENTAL_MAX);
BUILD_BUG_ON(ARRAY_SIZE(features) != XFS_EXPERIMENTAL_MAX);

1
fs/xfs/xfs_message.h
View File

@ -99,6 +99,7 @@ enum xfs_experimental_feat {
XFS_EXPERIMENTAL_EXCHRANGE,
XFS_EXPERIMENTAL_PPTR,
XFS_EXPERIMENTAL_METADIR,
XFS_EXPERIMENTAL_ZONED,
XFS_EXPERIMENTAL_MAX,
};

212
fs/xfs/xfs_mount.c
View File

@ -40,6 +40,7 @@
#include "xfs_rtrmap_btree.h"
#include "xfs_rtrefcount_btree.h"
#include "scrub/stats.h"
#include "xfs_zone_alloc.h"
static DEFINE_MUTEX(xfs_uuid_table_mutex);
static int xfs_uuid_table_size;
@ -461,22 +462,38 @@ xfs_mount_reset_sbqflags(
return xfs_sync_sb(mp, false);
}
uint64_t
xfs_default_resblks(xfs_mount_t *mp)
{
uint64_t resblks;
static const char *const xfs_free_pool_name[] = {
[XC_FREE_BLOCKS] = "free blocks",
[XC_FREE_RTEXTENTS] = "free rt extents",
[XC_FREE_RTAVAILABLE] = "available rt extents",
};
/*
* We default to 5% or 8192 fsbs of space reserved, whichever is
* smaller. This is intended to cover concurrent allocation
* transactions when we initially hit enospc. These each require a 4
* block reservation. Hence by default we cover roughly 2000 concurrent
* allocation reservations.
*/
resblks = mp->m_sb.sb_dblocks;
do_div(resblks, 20);
resblks = min_t(uint64_t, resblks, 8192);
return resblks;
uint64_t
xfs_default_resblks(
struct xfs_mount *mp,
enum xfs_free_counter ctr)
{
switch (ctr) {
case XC_FREE_BLOCKS:
/*
* Default to 5% or 8192 FSBs of space reserved, whichever is
* smaller.
*
* This is intended to cover concurrent allocation transactions
* when we initially hit ENOSPC. These each require a 4 block
* reservation. Hence by default we cover roughly 2000
* concurrent allocation reservations.
*/
return min(div_u64(mp->m_sb.sb_dblocks, 20), 8192ULL);
case XC_FREE_RTEXTENTS:
case XC_FREE_RTAVAILABLE:
if (IS_ENABLED(CONFIG_XFS_RT) && xfs_has_zoned(mp))
return xfs_zoned_default_resblks(mp, ctr);
return 0;
default:
ASSERT(0);
return 0;
}
}
/* Ensure the summary counts are correct. */
@ -543,7 +560,7 @@ xfs_check_summary_counts(
* If we're mounting the rt volume after recovering the log, recompute
* frextents from the rtbitmap file to fix the inconsistency.
*/
if (xfs_has_realtime(mp) && !xfs_is_clean(mp)) {
if (xfs_has_realtime(mp) && !xfs_has_zoned(mp) && !xfs_is_clean(mp)) {
error = xfs_rtalloc_reinit_frextents(mp);
if (error)
return error;
@ -678,6 +695,7 @@ xfs_mountfs(
uint quotamount = 0;
uint quotaflags = 0;
int error = 0;
int i;
xfs_sb_mount_common(mp, sbp);
@ -747,27 +765,15 @@ xfs_mountfs(
/* enable fail_at_unmount as default */
mp->m_fail_unmount = true;
super_set_sysfs_name_id(mp->m_super);
error = xfs_sysfs_init(&mp->m_kobj, &xfs_mp_ktype,
NULL, mp->m_super->s_id);
if (error)
goto out;
error = xfs_sysfs_init(&mp->m_stats.xs_kobj, &xfs_stats_ktype,
&mp->m_kobj, "stats");
if (error)
goto out_remove_sysfs;
xchk_stats_register(mp->m_scrub_stats, mp->m_debugfs);
error = xfs_error_sysfs_init(mp);
error = xfs_mount_sysfs_init(mp);
if (error)
goto out_remove_scrub_stats;
xchk_stats_register(mp->m_scrub_stats, mp->m_debugfs);
error = xfs_errortag_init(mp);
if (error)
goto out_remove_error_sysfs;
goto out_remove_sysfs;
error = xfs_uuid_mount(mp);
if (error)
@ -1031,6 +1037,12 @@ xfs_mountfs(
if (xfs_is_readonly(mp) && !xfs_has_norecovery(mp))
xfs_log_clean(mp);
if (xfs_has_zoned(mp)) {
error = xfs_mount_zones(mp);
if (error)
goto out_rtunmount;
}
/*
* Complete the quota initialisation, post-log-replay component.
*/
@ -1046,22 +1058,28 @@ xfs_mountfs(
* privileged transactions. This is needed so that transaction
* space required for critical operations can dip into this pool
* when at ENOSPC. This is needed for operations like create with
* attr, unwritten extent conversion at ENOSPC, etc. Data allocations
* are not allowed to use this reserved space.
* attr, unwritten extent conversion at ENOSPC, garbage collection
* etc. Data allocations are not allowed to use this reserved space.
*
* This may drive us straight to ENOSPC on mount, but that implies
* we were already there on the last unmount. Warn if this occurs.
*/
if (!xfs_is_readonly(mp)) {
error = xfs_reserve_blocks(mp, xfs_default_resblks(mp));
if (error)
xfs_warn(mp,
"Unable to allocate reserve blocks. Continuing without reserve pool.");
for (i = 0; i < XC_FREE_NR; i++) {
error = xfs_reserve_blocks(mp, i,
xfs_default_resblks(mp, i));
if (error)
xfs_warn(mp,
"Unable to allocate reserve blocks. Continuing without reserve pool for %s.",
xfs_free_pool_name[i]);
}
/* Reserve AG blocks for future btree expansion. */
error = xfs_fs_reserve_ag_blocks(mp);
if (error && error != -ENOSPC)
goto out_agresv;
xfs_zone_gc_start(mp);
}
return 0;
@ -1069,6 +1087,8 @@ xfs_mountfs(
out_agresv:
xfs_fs_unreserve_ag_blocks(mp);
xfs_qm_unmount_quotas(mp);
if (xfs_has_zoned(mp))
xfs_unmount_zones(mp);
out_rtunmount:
xfs_rtunmount_inodes(mp);
out_rele_rip:
@ -1116,13 +1136,10 @@ xfs_mountfs(
xfs_uuid_unmount(mp);
out_remove_errortag:
xfs_errortag_del(mp);
out_remove_error_sysfs:
xfs_error_sysfs_del(mp);
out_remove_sysfs:
xfs_mount_sysfs_del(mp);
out_remove_scrub_stats:
xchk_stats_unregister(mp->m_scrub_stats);
xfs_sysfs_del(&mp->m_stats.xs_kobj);
out_remove_sysfs:
xfs_sysfs_del(&mp->m_kobj);
out:
return error;
}
@ -1148,8 +1165,12 @@ xfs_unmountfs(
xfs_inodegc_flush(mp);
xfs_blockgc_stop(mp);
if (!test_bit(XFS_OPSTATE_READONLY, &mp->m_opstate))
xfs_zone_gc_stop(mp);
xfs_fs_unreserve_ag_blocks(mp);
xfs_qm_unmount_quotas(mp);
if (xfs_has_zoned(mp))
xfs_unmount_zones(mp);
xfs_rtunmount_inodes(mp);
xfs_irele(mp->m_rootip);
if (mp->m_metadirip)
@ -1173,7 +1194,7 @@ xfs_unmountfs(
* we only every apply deltas to the superblock and hence the incore
* value does not matter....
*/
error = xfs_reserve_blocks(mp, 0);
error = xfs_reserve_blocks(mp, XC_FREE_BLOCKS, 0);
if (error)
xfs_warn(mp, "Unable to free reserved block pool. "
"Freespace may not be correct on next mount.");
@ -1195,10 +1216,8 @@ xfs_unmountfs(
xfs_free_rtgroups(mp, 0, mp->m_sb.sb_rgcount);
xfs_free_perag_range(mp, 0, mp->m_sb.sb_agcount);
xfs_errortag_del(mp);
xfs_error_sysfs_del(mp);
xchk_stats_unregister(mp->m_scrub_stats);
xfs_sysfs_del(&mp->m_stats.xs_kobj);
xfs_sysfs_del(&mp->m_kobj);
xfs_mount_sysfs_del(mp);
}
/*
@ -1220,52 +1239,67 @@ xfs_fs_writable(
return true;
}
/*
* Estimate the amount of free space that is not available to userspace and is
* not explicitly reserved from the incore fdblocks. This includes:
*
* - The minimum number of blocks needed to support splitting a bmap btree
* - The blocks currently in use by the freespace btrees because they record
* the actual blocks that will fill per-AG metadata space reservations
*/
uint64_t
xfs_freecounter_unavailable(
struct xfs_mount *mp,
enum xfs_free_counter ctr)
{
if (ctr != XC_FREE_BLOCKS)
return 0;
return mp->m_alloc_set_aside + atomic64_read(&mp->m_allocbt_blks);
}
void
xfs_add_freecounter(
struct xfs_mount *mp,
struct percpu_counter *counter,
enum xfs_free_counter ctr,
uint64_t delta)
{
bool has_resv_pool = (counter == &mp->m_fdblocks);
struct xfs_freecounter *counter = &mp->m_free[ctr];
uint64_t res_used;
/*
* If the reserve pool is depleted, put blocks back into it first.
* Most of the time the pool is full.
*/
if (!has_resv_pool || mp->m_resblks == mp->m_resblks_avail) {
percpu_counter_add(counter, delta);
if (likely(counter->res_avail == counter->res_total)) {
percpu_counter_add(&counter->count, delta);
return;
}
spin_lock(&mp->m_sb_lock);
res_used = mp->m_resblks - mp->m_resblks_avail;
res_used = counter->res_total - counter->res_avail;
if (res_used > delta) {
mp->m_resblks_avail += delta;
counter->res_avail += delta;
} else {
delta -= res_used;
mp->m_resblks_avail = mp->m_resblks;
percpu_counter_add(counter, delta);
counter->res_avail = counter->res_total;
percpu_counter_add(&counter->count, delta);
}
spin_unlock(&mp->m_sb_lock);
}
/* Adjust in-core free blocks or RT extents. */
int
xfs_dec_freecounter(
struct xfs_mount *mp,
struct percpu_counter *counter,
enum xfs_free_counter ctr,
uint64_t delta,
bool rsvd)
{
int64_t lcounter;
uint64_t set_aside = 0;
struct xfs_freecounter *counter = &mp->m_free[ctr];
s32 batch;
bool has_resv_pool;
ASSERT(counter == &mp->m_fdblocks || counter == &mp->m_frextents);
has_resv_pool = (counter == &mp->m_fdblocks);
if (rsvd)
ASSERT(has_resv_pool);
ASSERT(ctr < XC_FREE_NR);
/*
* Taking blocks away, need to be more accurate the closer we
@ -1275,7 +1309,7 @@ xfs_dec_freecounter(
* then make everything serialise as we are real close to
* ENOSPC.
*/
if (__percpu_counter_compare(counter, 2 * XFS_FDBLOCKS_BATCH,
if (__percpu_counter_compare(&counter->count, 2 * XFS_FDBLOCKS_BATCH,
XFS_FDBLOCKS_BATCH) < 0)
batch = 1;
else
@ -1292,34 +1326,34 @@ xfs_dec_freecounter(
* problems (i.e. transaction abort, pagecache discards, etc.) than
* slightly premature -ENOSPC.
*/
if (has_resv_pool)
set_aside = xfs_fdblocks_unavailable(mp);
percpu_counter_add_batch(counter, -((int64_t)delta), batch);
if (__percpu_counter_compare(counter, set_aside,
XFS_FDBLOCKS_BATCH) >= 0) {
/* we had space! */
return 0;
}
/*
* lock up the sb for dipping into reserves before releasing the space
* that took us to ENOSPC.
*/
spin_lock(&mp->m_sb_lock);
percpu_counter_add(counter, delta);
if (!has_resv_pool || !rsvd)
goto fdblocks_enospc;
lcounter = (long long)mp->m_resblks_avail - delta;
if (lcounter >= 0) {
mp->m_resblks_avail = lcounter;
spin_unlock(&mp->m_sb_lock);
return 0;
}
xfs_warn_once(mp,
percpu_counter_add_batch(&counter->count, -((int64_t)delta), batch);
if (__percpu_counter_compare(&counter->count,
xfs_freecounter_unavailable(mp, ctr),
XFS_FDBLOCKS_BATCH) < 0) {
/*
* Lock up the sb for dipping into reserves before releasing the
* space that took us to ENOSPC.
*/
spin_lock(&mp->m_sb_lock);
percpu_counter_add(&counter->count, delta);
if (!rsvd)
goto fdblocks_enospc;
if (delta > counter->res_avail) {
if (ctr == XC_FREE_BLOCKS)
xfs_warn_once(mp,
"Reserve blocks depleted! Consider increasing reserve pool size.");
goto fdblocks_enospc;
}
counter->res_avail -= delta;
trace_xfs_freecounter_reserved(mp, ctr, delta, _RET_IP_);
spin_unlock(&mp->m_sb_lock);
}
/* we had space! */
return 0;
fdblocks_enospc:
trace_xfs_freecounter_enospc(mp, ctr, delta, _RET_IP_);
spin_unlock(&mp->m_sb_lock);
return -ENOSPC;
}

131
fs/xfs/xfs_mount.h
View File

@ -97,12 +97,42 @@ struct xfs_groups {
*/
uint8_t blklog;
/*
* Zoned devices can have gaps beyond the usable capacity of a zone and
* the end in the LBA/daddr address space. In other words, the hardware
* equivalent to the RT groups already takes care of the power of 2
* alignment for us. In this case the sparse FSB/RTB address space maps
* 1:1 to the device address space.
*/
bool has_daddr_gaps;
/*
* Mask to extract the group-relative block number from a FSB.
* For a pre-rtgroups filesystem we pretend to have one very large
* rtgroup, so this mask must be 64-bit.
*/
uint64_t blkmask;
/*
* Start of the first group in the device. This is used to support a
* RT device following the data device on the same block device for
* SMR hard drives.
*/
xfs_fsblock_t start_fsb;
};
struct xfs_freecounter {
/* free blocks for general use: */
struct percpu_counter count;
/* total reserved blocks: */
uint64_t res_total;
/* available reserved blocks: */
uint64_t res_avail;
/* reserved blks @ remount,ro: */
uint64_t res_saved;
};
/*
@ -198,6 +228,7 @@ typedef struct xfs_mount {
bool m_fail_unmount;
bool m_finobt_nores; /* no per-AG finobt resv. */
bool m_update_sb; /* sb needs update in mount */
unsigned int m_max_open_zones;
/*
* Bitsets of per-fs metadata that have been checked and/or are sick.
@ -222,8 +253,8 @@ typedef struct xfs_mount {
spinlock_t ____cacheline_aligned m_sb_lock; /* sb counter lock */
struct percpu_counter m_icount; /* allocated inodes counter */
struct percpu_counter m_ifree; /* free inodes counter */
struct percpu_counter m_fdblocks; /* free block counter */
struct percpu_counter m_frextents; /* free rt extent counter */
struct xfs_freecounter m_free[XC_FREE_NR];
/*
* Count of data device blocks reserved for delayed allocations,
@ -245,10 +276,8 @@ typedef struct xfs_mount {
atomic64_t m_allocbt_blks;
struct xfs_groups m_groups[XG_TYPE_MAX];
uint64_t m_resblks; /* total reserved blocks */
uint64_t m_resblks_avail;/* available reserved blocks */
uint64_t m_resblks_save; /* reserved blks @ remount,ro */
struct delayed_work m_reclaim_work; /* background inode reclaim */
struct xfs_zone_info *m_zone_info; /* zone allocator information */
struct dentry *m_debugfs; /* debugfs parent */
struct xfs_kobj m_kobj;
struct xfs_kobj m_error_kobj;
@ -258,10 +287,16 @@ typedef struct xfs_mount {
#ifdef CONFIG_XFS_ONLINE_SCRUB_STATS
struct xchk_stats *m_scrub_stats;
#endif
struct xfs_kobj m_zoned_kobj;
xfs_agnumber_t m_agfrotor; /* last ag where space found */
atomic_t m_agirotor; /* last ag dir inode alloced */
atomic_t m_rtgrotor; /* last rtgroup rtpicked */
struct mutex m_metafile_resv_lock;
uint64_t m_metafile_resv_target;
uint64_t m_metafile_resv_used;
uint64_t m_metafile_resv_avail;
/* Memory shrinker to throttle and reprioritize inodegc */
struct shrinker *m_inodegc_shrinker;
/*
@ -336,8 +371,10 @@ typedef struct xfs_mount {
#define XFS_FEAT_NREXT64 (1ULL << 26) /* large extent counters */
#define XFS_FEAT_EXCHANGE_RANGE (1ULL << 27) /* exchange range */
#define XFS_FEAT_METADIR (1ULL << 28) /* metadata directory tree */
#define XFS_FEAT_ZONED (1ULL << 29) /* zoned RT device */
/* Mount features */
#define XFS_FEAT_NOLIFETIME (1ULL << 47) /* disable lifetime hints */
#define XFS_FEAT_NOATTR2 (1ULL << 48) /* disable attr2 creation */
#define XFS_FEAT_NOALIGN (1ULL << 49) /* ignore alignment */
#define XFS_FEAT_ALLOCSIZE (1ULL << 50) /* user specified allocation size */
@ -392,6 +429,8 @@ __XFS_HAS_FEAT(needsrepair, NEEDSREPAIR)
__XFS_HAS_FEAT(large_extent_counts, NREXT64)
__XFS_HAS_FEAT(exchange_range, EXCHANGE_RANGE)
__XFS_HAS_FEAT(metadir, METADIR)
__XFS_HAS_FEAT(zoned, ZONED)
__XFS_HAS_FEAT(nolifetime, NOLIFETIME)
static inline bool xfs_has_rtgroups(const struct xfs_mount *mp)
{
@ -402,7 +441,9 @@ static inline bool xfs_has_rtgroups(const struct xfs_mount *mp)
static inline bool xfs_has_rtsb(const struct xfs_mount *mp)
{
/* all rtgroups filesystems with an rt section have an rtsb */
return xfs_has_rtgroups(mp) && xfs_has_realtime(mp);
return xfs_has_rtgroups(mp) &&
xfs_has_realtime(mp) &&
!xfs_has_zoned(mp);
}
static inline bool xfs_has_rtrmapbt(const struct xfs_mount *mp)
@ -417,6 +458,11 @@ static inline bool xfs_has_rtreflink(const struct xfs_mount *mp)
xfs_has_reflink(mp);
}
static inline bool xfs_has_nonzoned(const struct xfs_mount *mp)
{
return !xfs_has_zoned(mp);
}
/*
* Some features are always on for v5 file systems, allow the compiler to
* eliminiate dead code when building without v4 support.
@ -520,6 +566,10 @@ __XFS_HAS_FEAT(nouuid, NOUUID)
#define XFS_OPSTATE_WARNED_METADIR 17
/* Filesystem should use qflags to determine quotaon status */
#define XFS_OPSTATE_RESUMING_QUOTAON 18
/* Kernel has logged a warning about zoned RT device being used on this fs. */
#define XFS_OPSTATE_WARNED_ZONED 19
/* (Zoned) GC is in progress */
#define XFS_OPSTATE_ZONEGC_RUNNING 20
#define __XFS_IS_OPSTATE(name, NAME) \
static inline bool xfs_is_ ## name (struct xfs_mount *mp) \
@ -564,6 +614,7 @@ static inline bool xfs_clear_resuming_quotaon(struct xfs_mount *mp)
#endif /* CONFIG_XFS_QUOTA */
__XFS_IS_OPSTATE(done_with_log_incompat, UNSET_LOG_INCOMPAT)
__XFS_IS_OPSTATE(using_logged_xattrs, USE_LARP)
__XFS_IS_OPSTATE(zonegc_running, ZONEGC_RUNNING)
static inline bool
xfs_should_warn(struct xfs_mount *mp, long nr)
@ -633,7 +684,8 @@ xfs_daddr_to_agbno(struct xfs_mount *mp, xfs_daddr_t d)
}
extern void xfs_uuid_table_free(void);
extern uint64_t xfs_default_resblks(xfs_mount_t *mp);
uint64_t xfs_default_resblks(struct xfs_mount *mp,
enum xfs_free_counter ctr);
extern int xfs_mountfs(xfs_mount_t *mp);
extern void xfs_unmountfs(xfs_mount_t *);
@ -646,45 +698,74 @@ extern void xfs_unmountfs(xfs_mount_t *);
*/
#define XFS_FDBLOCKS_BATCH 1024
uint64_t xfs_freecounter_unavailable(struct xfs_mount *mp,
enum xfs_free_counter ctr);
/*
* Estimate the amount of free space that is not available to userspace and is
* not explicitly reserved from the incore fdblocks. This includes:
*
* - The minimum number of blocks needed to support splitting a bmap btree
* - The blocks currently in use by the freespace btrees because they record
* the actual blocks that will fill per-AG metadata space reservations
* Sum up the freecount, but never return negative values.
*/
static inline uint64_t
xfs_fdblocks_unavailable(
struct xfs_mount *mp)
static inline s64 xfs_sum_freecounter(struct xfs_mount *mp,
enum xfs_free_counter ctr)
{
return mp->m_alloc_set_aside + atomic64_read(&mp->m_allocbt_blks);
return percpu_counter_sum_positive(&mp->m_free[ctr].count);
}
int xfs_dec_freecounter(struct xfs_mount *mp, struct percpu_counter *counter,
/*
* Same as above, but does return negative values. Mostly useful for
* special cases like repair and tracing.
*/
static inline s64 xfs_sum_freecounter_raw(struct xfs_mount *mp,
enum xfs_free_counter ctr)
{
return percpu_counter_sum(&mp->m_free[ctr].count);
}
/*
* This just provides and estimate without the cpu-local updates, use
* xfs_sum_freecounter for the exact value.
*/
static inline s64 xfs_estimate_freecounter(struct xfs_mount *mp,
enum xfs_free_counter ctr)
{
return percpu_counter_read_positive(&mp->m_free[ctr].count);
}
static inline int xfs_compare_freecounter(struct xfs_mount *mp,
enum xfs_free_counter ctr, s64 rhs, s32 batch)
{
return __percpu_counter_compare(&mp->m_free[ctr].count, rhs, batch);
}
static inline void xfs_set_freecounter(struct xfs_mount *mp,
enum xfs_free_counter ctr, uint64_t val)
{
percpu_counter_set(&mp->m_free[ctr].count, val);
}
int xfs_dec_freecounter(struct xfs_mount *mp, enum xfs_free_counter ctr,
uint64_t delta, bool rsvd);
void xfs_add_freecounter(struct xfs_mount *mp, struct percpu_counter *counter,
void xfs_add_freecounter(struct xfs_mount *mp, enum xfs_free_counter ctr,
uint64_t delta);
static inline int xfs_dec_fdblocks(struct xfs_mount *mp, uint64_t delta,
bool reserved)
{
return xfs_dec_freecounter(mp, &mp->m_fdblocks, delta, reserved);
return xfs_dec_freecounter(mp, XC_FREE_BLOCKS, delta, reserved);
}
static inline void xfs_add_fdblocks(struct xfs_mount *mp, uint64_t delta)
{
xfs_add_freecounter(mp, &mp->m_fdblocks, delta);
xfs_add_freecounter(mp, XC_FREE_BLOCKS, delta);
}
static inline int xfs_dec_frextents(struct xfs_mount *mp, uint64_t delta)
{
return xfs_dec_freecounter(mp, &mp->m_frextents, delta, false);
return xfs_dec_freecounter(mp, XC_FREE_RTEXTENTS, delta, false);
}
static inline void xfs_add_frextents(struct xfs_mount *mp, uint64_t delta)
{
xfs_add_freecounter(mp, &mp->m_frextents, delta);
xfs_add_freecounter(mp, XC_FREE_RTEXTENTS, delta);
}
extern int xfs_readsb(xfs_mount_t *, int);
@ -706,5 +787,9 @@ int xfs_add_incompat_log_feature(struct xfs_mount *mp, uint32_t feature);
bool xfs_clear_incompat_log_features(struct xfs_mount *mp);
void xfs_mod_delalloc(struct xfs_inode *ip, int64_t data_delta,
int64_t ind_delta);
static inline void xfs_mod_sb_delalloc(struct xfs_mount *mp, int64_t delta)
{
percpu_counter_add(&mp->m_delalloc_blks, delta);
}
#endif /* __XFS_MOUNT_H__ */

3
fs/xfs/xfs_qm.c
View File

@ -1711,7 +1711,8 @@ xfs_qm_mount_quotas(
* immediately. We only support rtquota if rtgroups are enabled to
* avoid problems with older kernels.
*/
if (mp->m_sb.sb_rextents && !xfs_has_rtgroups(mp)) {
if (mp->m_sb.sb_rextents &&
(!xfs_has_rtgroups(mp) || xfs_has_zoned(mp))) {
xfs_notice(mp, "Cannot turn on quotas for realtime filesystem");
mp->m_qflags = 0;
goto write_changes;

18
fs/xfs/xfs_reflink.c
View File

@ -235,7 +235,7 @@ xfs_reflink_trim_around_shared(
int error = 0;
/* Holes, unwritten, and delalloc extents cannot be shared */
if (!xfs_is_cow_inode(ip) || !xfs_bmap_is_written_extent(irec)) {
if (!xfs_is_reflink_inode(ip) || !xfs_bmap_is_written_extent(irec)) {
*shared = false;
return 0;
}
@ -651,7 +651,7 @@ xfs_reflink_cancel_cow_blocks(
if (isnullstartblock(del.br_startblock)) {
xfs_bmap_del_extent_delay(ip, XFS_COW_FORK, &icur, &got,
&del);
&del, 0);
} else if (del.br_state == XFS_EXT_UNWRITTEN || cancel_real) {
ASSERT((*tpp)->t_highest_agno == NULLAGNUMBER);
@ -1207,15 +1207,9 @@ xfs_reflink_ag_has_free_space(
if (!xfs_has_rmapbt(mp))
return 0;
if (XFS_IS_REALTIME_INODE(ip)) {
struct xfs_rtgroup *rtg;
xfs_rgnumber_t rgno;
rgno = xfs_rtb_to_rgno(mp, fsb);
rtg = xfs_rtgroup_get(mp, rgno);
if (xfs_metafile_resv_critical(rtg_rmap(rtg)))
error = -ENOSPC;
xfs_rtgroup_put(rtg);
return error;
if (xfs_metafile_resv_critical(mp))
return -ENOSPC;
return 0;
}
agno = XFS_FSB_TO_AGNO(mp, fsb);
@ -1538,7 +1532,7 @@ xfs_reflink_zero_posteof(
return 0;
trace_xfs_zero_eof(ip, isize, pos - isize);
return xfs_zero_range(ip, isize, pos - isize, NULL);
return xfs_zero_range(ip, isize, pos - isize, NULL, NULL);
}
/*

237
fs/xfs/xfs_rtalloc.c
View File

@ -33,6 +33,7 @@
#include "xfs_trace.h"
#include "xfs_rtrefcount_btree.h"
#include "xfs_reflink.h"
#include "xfs_zone_alloc.h"
/*
* Return whether there are any free extents in the size range given
@ -663,7 +664,8 @@ xfs_rtunmount_rtg(
for (i = 0; i < XFS_RTGI_MAX; i++)
xfs_rtginode_irele(&rtg->rtg_inodes[i]);
kvfree(rtg->rtg_rsum_cache);
if (!xfs_has_zoned(rtg_mount(rtg)))
kvfree(rtg->rtg_rsum_cache);
}
static int
@ -858,6 +860,84 @@ xfs_growfs_rt_init_rtsb(
return error;
}
static void
xfs_growfs_rt_sb_fields(
struct xfs_trans *tp,
const struct xfs_mount *nmp)
{
struct xfs_mount *mp = tp->t_mountp;
if (nmp->m_sb.sb_rextsize != mp->m_sb.sb_rextsize)
xfs_trans_mod_sb(tp, XFS_TRANS_SB_REXTSIZE,
nmp->m_sb.sb_rextsize - mp->m_sb.sb_rextsize);
if (nmp->m_sb.sb_rbmblocks != mp->m_sb.sb_rbmblocks)
xfs_trans_mod_sb(tp, XFS_TRANS_SB_RBMBLOCKS,
nmp->m_sb.sb_rbmblocks - mp->m_sb.sb_rbmblocks);
if (nmp->m_sb.sb_rblocks != mp->m_sb.sb_rblocks)
xfs_trans_mod_sb(tp, XFS_TRANS_SB_RBLOCKS,
nmp->m_sb.sb_rblocks - mp->m_sb.sb_rblocks);
if (nmp->m_sb.sb_rextents != mp->m_sb.sb_rextents)
xfs_trans_mod_sb(tp, XFS_TRANS_SB_REXTENTS,
nmp->m_sb.sb_rextents - mp->m_sb.sb_rextents);
if (nmp->m_sb.sb_rextslog != mp->m_sb.sb_rextslog)
xfs_trans_mod_sb(tp, XFS_TRANS_SB_REXTSLOG,
nmp->m_sb.sb_rextslog - mp->m_sb.sb_rextslog);
if (nmp->m_sb.sb_rgcount != mp->m_sb.sb_rgcount)
xfs_trans_mod_sb(tp, XFS_TRANS_SB_RGCOUNT,
nmp->m_sb.sb_rgcount - mp->m_sb.sb_rgcount);
}
static int
xfs_growfs_rt_zoned(
struct xfs_rtgroup *rtg,
xfs_rfsblock_t nrblocks)
{
struct xfs_mount *mp = rtg_mount(rtg);
struct xfs_mount *nmp;
struct xfs_trans *tp;
xfs_rtbxlen_t freed_rtx;
int error;
/*
* Calculate new sb and mount fields for this round. Also ensure the
* rtg_extents value is uptodate as the rtbitmap code relies on it.
*/
nmp = xfs_growfs_rt_alloc_fake_mount(mp, nrblocks,
mp->m_sb.sb_rextsize);
if (!nmp)
return -ENOMEM;
freed_rtx = nmp->m_sb.sb_rextents - mp->m_sb.sb_rextents;
xfs_rtgroup_calc_geometry(nmp, rtg, rtg_rgno(rtg),
nmp->m_sb.sb_rgcount, nmp->m_sb.sb_rextents);
error = xfs_trans_alloc(mp, &M_RES(nmp)->tr_growrtfree, 0, 0, 0, &tp);
if (error)
goto out_free;
xfs_rtgroup_lock(rtg, XFS_RTGLOCK_RMAP);
xfs_rtgroup_trans_join(tp, rtg, XFS_RTGLOCK_RMAP);
xfs_growfs_rt_sb_fields(tp, nmp);
xfs_trans_mod_sb(tp, XFS_TRANS_SB_FREXTENTS, freed_rtx);
error = xfs_trans_commit(tp);
if (error)
goto out_free;
/*
* Ensure the mount RT feature flag is now set, and compute new
* maxlevels for rt btrees.
*/
mp->m_features |= XFS_FEAT_REALTIME;
xfs_rtrmapbt_compute_maxlevels(mp);
xfs_rtrefcountbt_compute_maxlevels(mp);
xfs_zoned_add_available(mp, freed_rtx);
out_free:
kfree(nmp);
return error;
}
static int
xfs_growfs_rt_bmblock(
struct xfs_rtgroup *rtg,
@ -943,24 +1023,7 @@ xfs_growfs_rt_bmblock(
/*
* Update superblock fields.
*/
if (nmp->m_sb.sb_rextsize != mp->m_sb.sb_rextsize)
xfs_trans_mod_sb(args.tp, XFS_TRANS_SB_REXTSIZE,
nmp->m_sb.sb_rextsize - mp->m_sb.sb_rextsize);
if (nmp->m_sb.sb_rbmblocks != mp->m_sb.sb_rbmblocks)
xfs_trans_mod_sb(args.tp, XFS_TRANS_SB_RBMBLOCKS,
nmp->m_sb.sb_rbmblocks - mp->m_sb.sb_rbmblocks);
if (nmp->m_sb.sb_rblocks != mp->m_sb.sb_rblocks)
xfs_trans_mod_sb(args.tp, XFS_TRANS_SB_RBLOCKS,
nmp->m_sb.sb_rblocks - mp->m_sb.sb_rblocks);
if (nmp->m_sb.sb_rextents != mp->m_sb.sb_rextents)
xfs_trans_mod_sb(args.tp, XFS_TRANS_SB_REXTENTS,
nmp->m_sb.sb_rextents - mp->m_sb.sb_rextents);
if (nmp->m_sb.sb_rextslog != mp->m_sb.sb_rextslog)
xfs_trans_mod_sb(args.tp, XFS_TRANS_SB_REXTSLOG,
nmp->m_sb.sb_rextslog - mp->m_sb.sb_rextslog);
if (nmp->m_sb.sb_rgcount != mp->m_sb.sb_rgcount)
xfs_trans_mod_sb(args.tp, XFS_TRANS_SB_RGCOUNT,
nmp->m_sb.sb_rgcount - mp->m_sb.sb_rgcount);
xfs_growfs_rt_sb_fields(args.tp, nmp);
/*
* Free the new extent.
@ -1127,6 +1190,11 @@ xfs_growfs_rtg(
goto out_rele;
}
if (xfs_has_zoned(mp)) {
error = xfs_growfs_rt_zoned(rtg, nrblocks);
goto out_rele;
}
error = xfs_growfs_rt_alloc_blocks(rtg, nrblocks, rextsize, &bmblocks);
if (error)
goto out_rele;
@ -1146,8 +1214,7 @@ xfs_growfs_rtg(
if (old_rsum_cache)
kvfree(old_rsum_cache);
xfs_rtgroup_rele(rtg);
return 0;
goto out_rele;
out_error:
/*
@ -1195,6 +1262,22 @@ xfs_growfs_check_rtgeom(
if (min_logfsbs > mp->m_sb.sb_logblocks)
return -EINVAL;
if (xfs_has_zoned(mp)) {
uint32_t gblocks = mp->m_groups[XG_TYPE_RTG].blocks;
uint32_t rem;
if (rextsize != 1)
return -EINVAL;
div_u64_rem(mp->m_sb.sb_rblocks, gblocks, &rem);
if (rem) {
xfs_warn(mp,
"new RT volume size (%lld) not aligned to RT group size (%d)",
mp->m_sb.sb_rblocks, gblocks);
return -EINVAL;
}
}
return 0;
}
@ -1248,6 +1331,35 @@ xfs_grow_last_rtg(
mp->m_sb.sb_rgextents;
}
/*
* Read in the last block of the RT device to make sure it is accessible.
*/
static int
xfs_rt_check_size(
struct xfs_mount *mp,
xfs_rfsblock_t last_block)
{
xfs_daddr_t daddr = XFS_FSB_TO_BB(mp, last_block);
struct xfs_buf *bp;
int error;
if (XFS_BB_TO_FSB(mp, daddr) != last_block) {
xfs_warn(mp, "RT device size overflow: %llu != %llu",
XFS_BB_TO_FSB(mp, daddr), last_block);
return -EFBIG;
}
error = xfs_buf_read_uncached(mp->m_rtdev_targp,
XFS_FSB_TO_BB(mp, mp->m_sb.sb_rtstart) + daddr,
XFS_FSB_TO_BB(mp, 1), 0, &bp, NULL);
if (error)
xfs_warn(mp, "cannot read last RT device sector (%lld)",
last_block);
else
xfs_buf_relse(bp);
return error;
}
/*
* Grow the realtime area of the filesystem.
*/
@ -1259,7 +1371,6 @@ xfs_growfs_rt(
xfs_rgnumber_t old_rgcount = mp->m_sb.sb_rgcount;
xfs_rgnumber_t new_rgcount = 1;
xfs_rgnumber_t rgno;
struct xfs_buf *bp;
xfs_agblock_t old_rextsize = mp->m_sb.sb_rextsize;
int error;
@ -1302,15 +1413,10 @@ xfs_growfs_rt(
error = xfs_sb_validate_fsb_count(&mp->m_sb, in->newblocks);
if (error)
goto out_unlock;
/*
* Read in the last block of the device, make sure it exists.
*/
error = xfs_buf_read_uncached(mp->m_rtdev_targp,
XFS_FSB_TO_BB(mp, in->newblocks - 1),
XFS_FSB_TO_BB(mp, 1), 0, &bp, NULL);
error = xfs_rt_check_size(mp, in->newblocks - 1);
if (error)
goto out_unlock;
xfs_buf_relse(bp);
/*
* Calculate new parameters. These are the final values to be reached.
@ -1376,8 +1482,7 @@ xfs_growfs_rt(
error = error2;
/* Reset the rt metadata btree space reservations. */
xfs_rt_resv_free(mp);
error2 = xfs_rt_resv_init(mp);
error2 = xfs_metafile_resv_init(mp);
if (error2 && error2 != -ENOSPC)
error = error2;
}
@ -1444,10 +1549,6 @@ int /* error */
xfs_rtmount_init(
struct xfs_mount *mp) /* file system mount structure */
{
struct xfs_buf *bp; /* buffer for last block of subvolume */
xfs_daddr_t d; /* address of last block of subvolume */
int error;
if (mp->m_sb.sb_rblocks == 0)
return 0;
if (mp->m_rtdev_targp == NULL) {
@ -1458,25 +1559,7 @@ xfs_rtmount_init(
mp->m_rsumblocks = xfs_rtsummary_blockcount(mp, &mp->m_rsumlevels);
/*
* Check that the realtime section is an ok size.
*/
d = (xfs_daddr_t)XFS_FSB_TO_BB(mp, mp->m_sb.sb_rblocks);
if (XFS_BB_TO_FSB(mp, d) != mp->m_sb.sb_rblocks) {
xfs_warn(mp, "realtime mount -- %llu != %llu",
(unsigned long long) XFS_BB_TO_FSB(mp, d),
(unsigned long long) mp->m_sb.sb_rblocks);
return -EFBIG;
}
error = xfs_buf_read_uncached(mp->m_rtdev_targp,
d - XFS_FSB_TO_BB(mp, 1),
XFS_FSB_TO_BB(mp, 1), 0, &bp, NULL);
if (error) {
xfs_warn(mp, "realtime device size check failed");
return error;
}
xfs_buf_relse(bp);
return 0;
return xfs_rt_check_size(mp, mp->m_sb.sb_rblocks - 1);
}
static int
@ -1519,50 +1602,10 @@ xfs_rtalloc_reinit_frextents(
spin_lock(&mp->m_sb_lock);
mp->m_sb.sb_frextents = val;
spin_unlock(&mp->m_sb_lock);
percpu_counter_set(&mp->m_frextents, mp->m_sb.sb_frextents);
xfs_set_freecounter(mp, XC_FREE_RTEXTENTS, mp->m_sb.sb_frextents);
return 0;
}
/* Free space reservations for rt metadata inodes. */
void
xfs_rt_resv_free(
struct xfs_mount *mp)
{
struct xfs_rtgroup *rtg = NULL;
unsigned int i;
while ((rtg = xfs_rtgroup_next(mp, rtg))) {
for (i = 0; i < XFS_RTGI_MAX; i++)
xfs_metafile_resv_free(rtg->rtg_inodes[i]);
}
}
/* Reserve space for rt metadata inodes' space expansion. */
int
xfs_rt_resv_init(
struct xfs_mount *mp)
{
struct xfs_rtgroup *rtg = NULL;
xfs_filblks_t ask;
int error = 0;
while ((rtg = xfs_rtgroup_next(mp, rtg))) {
int err2;
ask = xfs_rtrmapbt_calc_reserves(mp);
err2 = xfs_metafile_resv_init(rtg_rmap(rtg), ask);
if (err2 && !error)
error = err2;
ask = xfs_rtrefcountbt_calc_reserves(mp);
err2 = xfs_metafile_resv_init(rtg_refcount(rtg), ask);
if (err2 && !error)
error = err2;
}
return error;
}
/*
* Read in the bmbt of an rt metadata inode so that we never have to load them
* at runtime. This enables the use of shared ILOCKs for rtbitmap scans. Use
@ -1613,6 +1656,8 @@ xfs_rtmount_rtg(
}
}
if (xfs_has_zoned(mp))
return 0;
return xfs_alloc_rsum_cache(rtg, mp->m_sb.sb_rbmblocks);
}
@ -2097,6 +2142,8 @@ xfs_bmap_rtalloc(
ap->datatype & XFS_ALLOC_INITIAL_USER_DATA;
int error;
ASSERT(!xfs_has_zoned(ap->tp->t_mountp));
retry:
error = xfs_rtallocate_align(ap, &ralen, &raminlen, &prod, &noalign);
if (error)

5
fs/xfs/xfs_rtalloc.h
View File

@ -34,9 +34,6 @@ int /* error */
xfs_rtmount_inodes(
struct xfs_mount *mp); /* file system mount structure */
void xfs_rt_resv_free(struct xfs_mount *mp);
int xfs_rt_resv_init(struct xfs_mount *mp);
/*
* Grow the realtime area of the filesystem.
*/
@ -65,8 +62,6 @@ xfs_rtmount_init(
}
# define xfs_rtmount_inodes(m) (((mp)->m_sb.sb_rblocks == 0)? 0 : (-ENOSYS))
# define xfs_rtunmount_inodes(m)
# define xfs_rt_resv_free(mp) ((void)0)
# define xfs_rt_resv_init(mp) (0)
static inline int
xfs_growfs_check_rtgeom(const struct xfs_mount *mp,

165
fs/xfs/xfs_super.c
View File

@ -46,6 +46,7 @@
#include "xfs_exchmaps_item.h"
#include "xfs_parent.h"
#include "xfs_rtalloc.h"
#include "xfs_zone_alloc.h"
#include "scrub/stats.h"
#include "scrub/rcbag_btree.h"
@ -109,7 +110,8 @@ enum {
Opt_filestreams, Opt_quota, Opt_noquota, Opt_usrquota, Opt_grpquota,
Opt_prjquota, Opt_uquota, Opt_gquota, Opt_pquota,
Opt_uqnoenforce, Opt_gqnoenforce, Opt_pqnoenforce, Opt_qnoenforce,
Opt_discard, Opt_nodiscard, Opt_dax, Opt_dax_enum,
Opt_discard, Opt_nodiscard, Opt_dax, Opt_dax_enum, Opt_max_open_zones,
Opt_lifetime, Opt_nolifetime,
};
static const struct fs_parameter_spec xfs_fs_parameters[] = {
@ -154,6 +156,9 @@ static const struct fs_parameter_spec xfs_fs_parameters[] = {
fsparam_flag("nodiscard", Opt_nodiscard),
fsparam_flag("dax", Opt_dax),
fsparam_enum("dax", Opt_dax_enum, dax_param_enums),
fsparam_u32("max_open_zones", Opt_max_open_zones),
fsparam_flag("lifetime", Opt_lifetime),
fsparam_flag("nolifetime", Opt_nolifetime),
{}
};
@ -182,6 +187,7 @@ xfs_fs_show_options(
{ XFS_FEAT_LARGE_IOSIZE, ",largeio" },
{ XFS_FEAT_DAX_ALWAYS, ",dax=always" },
{ XFS_FEAT_DAX_NEVER, ",dax=never" },
{ XFS_FEAT_NOLIFETIME, ",nolifetime" },
{ 0, NULL }
};
struct xfs_mount *mp = XFS_M(root->d_sb);
@ -233,6 +239,9 @@ xfs_fs_show_options(
if (!(mp->m_qflags & XFS_ALL_QUOTA_ACCT))
seq_puts(m, ",noquota");
if (mp->m_max_open_zones)
seq_printf(m, ",max_open_zones=%u", mp->m_max_open_zones);
return 0;
}
@ -533,7 +542,15 @@ xfs_setup_devices(
if (error)
return error;
}
if (mp->m_rtdev_targp) {
if (mp->m_sb.sb_rtstart) {
if (mp->m_rtdev_targp) {
xfs_warn(mp,
"can't use internal and external rtdev at the same time");
return -EINVAL;
}
mp->m_rtdev_targp = mp->m_ddev_targp;
} else if (mp->m_rtname) {
error = xfs_setsize_buftarg(mp->m_rtdev_targp,
mp->m_sb.sb_sectsize);
if (error)
@ -757,7 +774,7 @@ xfs_mount_free(
{
if (mp->m_logdev_targp && mp->m_logdev_targp != mp->m_ddev_targp)
xfs_free_buftarg(mp->m_logdev_targp);
if (mp->m_rtdev_targp)
if (mp->m_rtdev_targp && mp->m_rtdev_targp != mp->m_ddev_targp)
xfs_free_buftarg(mp->m_rtdev_targp);
if (mp->m_ddev_targp)
xfs_free_buftarg(mp->m_ddev_targp);
@ -814,6 +831,7 @@ xfs_fs_sync_fs(
if (sb->s_writers.frozen == SB_FREEZE_PAGEFAULT) {
xfs_inodegc_stop(mp);
xfs_blockgc_stop(mp);
xfs_zone_gc_stop(mp);
}
return 0;
@ -834,10 +852,12 @@ xfs_statfs_data(
struct kstatfs *st)
{
int64_t fdblocks =
percpu_counter_sum(&mp->m_fdblocks);
xfs_sum_freecounter(mp, XC_FREE_BLOCKS);
/* make sure st->f_bfree does not underflow */
st->f_bfree = max(0LL, fdblocks - xfs_fdblocks_unavailable(mp));
st->f_bfree = max(0LL,
fdblocks - xfs_freecounter_unavailable(mp, XC_FREE_BLOCKS));
/*
* sb_dblocks can change during growfs, but nothing cares about reporting
* the old or new value during growfs.
@ -856,8 +876,9 @@ xfs_statfs_rt(
struct kstatfs *st)
{
st->f_bfree = xfs_rtbxlen_to_blen(mp,
percpu_counter_sum_positive(&mp->m_frextents));
st->f_blocks = mp->m_sb.sb_rblocks;
xfs_sum_freecounter(mp, XC_FREE_RTEXTENTS));
st->f_blocks = mp->m_sb.sb_rblocks - xfs_rtbxlen_to_blen(mp,
mp->m_free[XC_FREE_RTEXTENTS].res_total);
}
static void
@ -922,24 +943,32 @@ xfs_fs_statfs(
}
STATIC void
xfs_save_resvblks(struct xfs_mount *mp)
xfs_save_resvblks(
struct xfs_mount *mp)
{
mp->m_resblks_save = mp->m_resblks;
xfs_reserve_blocks(mp, 0);
enum xfs_free_counter i;
for (i = 0; i < XC_FREE_NR; i++) {
mp->m_free[i].res_saved = mp->m_free[i].res_total;
xfs_reserve_blocks(mp, i, 0);
}
}
STATIC void
xfs_restore_resvblks(struct xfs_mount *mp)
xfs_restore_resvblks(
struct xfs_mount *mp)
{
uint64_t resblks;
uint64_t resblks;
enum xfs_free_counter i;
if (mp->m_resblks_save) {
resblks = mp->m_resblks_save;
mp->m_resblks_save = 0;
} else
resblks = xfs_default_resblks(mp);
xfs_reserve_blocks(mp, resblks);
for (i = 0; i < XC_FREE_NR; i++) {
if (mp->m_free[i].res_saved) {
resblks = mp->m_free[i].res_saved;
mp->m_free[i].res_saved = 0;
} else
resblks = xfs_default_resblks(mp, i);
xfs_reserve_blocks(mp, i, resblks);
}
}
/*
@ -976,6 +1005,7 @@ xfs_fs_freeze(
if (ret && !xfs_is_readonly(mp)) {
xfs_blockgc_start(mp);
xfs_inodegc_start(mp);
xfs_zone_gc_start(mp);
}
return ret;
@ -997,6 +1027,7 @@ xfs_fs_unfreeze(
* filesystem.
*/
if (!xfs_is_readonly(mp)) {
xfs_zone_gc_start(mp);
xfs_blockgc_start(mp);
xfs_inodegc_start(mp);
}
@ -1058,6 +1089,19 @@ xfs_finish_flags(
return -EINVAL;
}
if (!xfs_has_zoned(mp)) {
if (mp->m_max_open_zones) {
xfs_warn(mp,
"max_open_zones mount option only supported on zoned file systems.");
return -EINVAL;
}
if (mp->m_features & XFS_FEAT_NOLIFETIME) {
xfs_warn(mp,
"nolifetime mount option only supported on zoned file systems.");
return -EINVAL;
}
}
return 0;
}
@ -1065,7 +1109,8 @@ static int
xfs_init_percpu_counters(
struct xfs_mount *mp)
{
int error;
int error;
int i;
error = percpu_counter_init(&mp->m_icount, 0, GFP_KERNEL);
if (error)
@ -1075,30 +1120,29 @@ xfs_init_percpu_counters(
if (error)
goto free_icount;
error = percpu_counter_init(&mp->m_fdblocks, 0, GFP_KERNEL);
if (error)
goto free_ifree;
error = percpu_counter_init(&mp->m_delalloc_blks, 0, GFP_KERNEL);
if (error)
goto free_fdblocks;
goto free_ifree;
error = percpu_counter_init(&mp->m_delalloc_rtextents, 0, GFP_KERNEL);
if (error)
goto free_delalloc;
error = percpu_counter_init(&mp->m_frextents, 0, GFP_KERNEL);
if (error)
goto free_delalloc_rt;
for (i = 0; i < XC_FREE_NR; i++) {
error = percpu_counter_init(&mp->m_free[i].count, 0,
GFP_KERNEL);
if (error)
goto free_freecounters;
}
return 0;
free_delalloc_rt:
free_freecounters:
while (--i > 0)
percpu_counter_destroy(&mp->m_free[i].count);
percpu_counter_destroy(&mp->m_delalloc_rtextents);
free_delalloc:
percpu_counter_destroy(&mp->m_delalloc_blks);
free_fdblocks:
percpu_counter_destroy(&mp->m_fdblocks);
free_ifree:
percpu_counter_destroy(&mp->m_ifree);
free_icount:
@ -1112,24 +1156,28 @@ xfs_reinit_percpu_counters(
{
percpu_counter_set(&mp->m_icount, mp->m_sb.sb_icount);
percpu_counter_set(&mp->m_ifree, mp->m_sb.sb_ifree);
percpu_counter_set(&mp->m_fdblocks, mp->m_sb.sb_fdblocks);
percpu_counter_set(&mp->m_frextents, mp->m_sb.sb_frextents);
xfs_set_freecounter(mp, XC_FREE_BLOCKS, mp->m_sb.sb_fdblocks);
if (!xfs_has_zoned(mp))
xfs_set_freecounter(mp, XC_FREE_RTEXTENTS,
mp->m_sb.sb_frextents);
}
static void
xfs_destroy_percpu_counters(
struct xfs_mount *mp)
{
enum xfs_free_counter i;
for (i = 0; i < XC_FREE_NR; i++)
percpu_counter_destroy(&mp->m_free[i].count);
percpu_counter_destroy(&mp->m_icount);
percpu_counter_destroy(&mp->m_ifree);
percpu_counter_destroy(&mp->m_fdblocks);
ASSERT(xfs_is_shutdown(mp) ||
percpu_counter_sum(&mp->m_delalloc_rtextents) == 0);
percpu_counter_destroy(&mp->m_delalloc_rtextents);
ASSERT(xfs_is_shutdown(mp) ||
percpu_counter_sum(&mp->m_delalloc_blks) == 0);
percpu_counter_destroy(&mp->m_delalloc_blks);
percpu_counter_destroy(&mp->m_frextents);
}
static int
@ -1210,6 +1258,18 @@ xfs_fs_shutdown(
xfs_force_shutdown(XFS_M(sb), SHUTDOWN_DEVICE_REMOVED);
}
static int
xfs_fs_show_stats(
struct seq_file *m,
struct dentry *root)
{
struct xfs_mount *mp = XFS_M(root->d_sb);
if (xfs_has_zoned(mp) && IS_ENABLED(CONFIG_XFS_RT))
xfs_zoned_show_stats(m, mp);
return 0;
}
static const struct super_operations xfs_super_operations = {
.alloc_inode = xfs_fs_alloc_inode,
.destroy_inode = xfs_fs_destroy_inode,
@ -1224,6 +1284,7 @@ static const struct super_operations xfs_super_operations = {
.nr_cached_objects = xfs_fs_nr_cached_objects,
.free_cached_objects = xfs_fs_free_cached_objects,
.shutdown = xfs_fs_shutdown,
.show_stats = xfs_fs_show_stats,
};
static int
@ -1436,6 +1497,15 @@ xfs_fs_parse_param(
xfs_fs_warn_deprecated(fc, param, XFS_FEAT_NOATTR2, true);
parsing_mp->m_features |= XFS_FEAT_NOATTR2;
return 0;
case Opt_max_open_zones:
parsing_mp->m_max_open_zones = result.uint_32;
return 0;
case Opt_lifetime:
parsing_mp->m_features &= ~XFS_FEAT_NOLIFETIME;
return 0;
case Opt_nolifetime:
parsing_mp->m_features |= XFS_FEAT_NOLIFETIME;
return 0;
default:
xfs_warn(parsing_mp, "unknown mount option [%s].", param->key);
return -EINVAL;
@ -1780,8 +1850,17 @@ xfs_fs_fill_super(
mp->m_features &= ~XFS_FEAT_DISCARD;
}
if (xfs_has_metadir(mp))
if (xfs_has_zoned(mp)) {
if (!xfs_has_metadir(mp)) {
xfs_alert(mp,
"metadir feature required for zoned realtime devices.");
error = -EINVAL;
goto out_filestream_unmount;
}
xfs_warn_experimental(mp, XFS_EXPERIMENTAL_ZONED);
} else if (xfs_has_metadir(mp)) {
xfs_warn_experimental(mp, XFS_EXPERIMENTAL_METADIR);
}
if (xfs_has_reflink(mp)) {
if (xfs_has_realtime(mp) &&
@ -1793,6 +1872,13 @@ xfs_fs_fill_super(
goto out_filestream_unmount;
}
if (xfs_has_zoned(mp)) {
xfs_alert(mp,
"reflink not compatible with zoned RT device!");
error = -EINVAL;
goto out_filestream_unmount;
}
if (xfs_globals.always_cow) {
xfs_info(mp, "using DEBUG-only always_cow mode.");
mp->m_always_cow = true;
@ -1917,6 +2003,9 @@ xfs_remount_rw(
/* Re-enable the background inode inactivation worker. */
xfs_inodegc_start(mp);
/* Restart zone reclaim */
xfs_zone_gc_start(mp);
return 0;
}
@ -1961,6 +2050,9 @@ xfs_remount_ro(
*/
xfs_inodegc_stop(mp);
/* Stop zone reclaim */
xfs_zone_gc_stop(mp);
/* Free the per-AG metadata reservation pool. */
xfs_fs_unreserve_ag_blocks(mp);
@ -2082,6 +2174,7 @@ xfs_init_fs_context(
for (i = 0; i < XG_TYPE_MAX; i++)
xa_init(&mp->m_groups[i].xa);
mutex_init(&mp->m_growlock);
mutex_init(&mp->m_metafile_resv_lock);
INIT_WORK(&mp->m_flush_inodes_work, xfs_flush_inodes_worker);
INIT_DELAYED_WORK(&mp->m_reclaim_work, xfs_reclaim_worker);
mp->m_kobj.kobject.kset = xfs_kset;

75
fs/xfs/xfs_sysfs.c
View File

@ -13,6 +13,7 @@
#include "xfs_log.h"
#include "xfs_log_priv.h"
#include "xfs_mount.h"
#include "xfs_zones.h"
struct xfs_sysfs_attr {
struct attribute attr;
@ -69,7 +70,7 @@ static struct attribute *xfs_mp_attrs[] = {
};
ATTRIBUTE_GROUPS(xfs_mp);
const struct kobj_type xfs_mp_ktype = {
static const struct kobj_type xfs_mp_ktype = {
.release = xfs_sysfs_release,
.sysfs_ops = &xfs_sysfs_ops,
.default_groups = xfs_mp_groups,
@ -701,45 +702,103 @@ xfs_error_sysfs_init_class(
return error;
}
static inline struct xfs_mount *zoned_to_mp(struct kobject *kobj)
{
return container_of(to_kobj(kobj), struct xfs_mount, m_zoned_kobj);
}
static ssize_t
max_open_zones_show(
struct kobject *kobj,
char *buf)
{
/* only report the open zones available for user data */
return sysfs_emit(buf, "%u\n",
zoned_to_mp(kobj)->m_max_open_zones - XFS_OPEN_GC_ZONES);
}
XFS_SYSFS_ATTR_RO(max_open_zones);
static struct attribute *xfs_zoned_attrs[] = {
ATTR_LIST(max_open_zones),
NULL,
};
ATTRIBUTE_GROUPS(xfs_zoned);
static const struct kobj_type xfs_zoned_ktype = {
.release = xfs_sysfs_release,
.sysfs_ops = &xfs_sysfs_ops,
.default_groups = xfs_zoned_groups,
};
int
xfs_error_sysfs_init(
xfs_mount_sysfs_init(
struct xfs_mount *mp)
{
int error;
super_set_sysfs_name_id(mp->m_super);
/* .../xfs/<dev>/ */
error = xfs_sysfs_init(&mp->m_kobj, &xfs_mp_ktype,
NULL, mp->m_super->s_id);
if (error)
return error;
/* .../xfs/<dev>/stats/ */
error = xfs_sysfs_init(&mp->m_stats.xs_kobj, &xfs_stats_ktype,
&mp->m_kobj, "stats");
if (error)
goto out_remove_fsdir;
/* .../xfs/<dev>/error/ */
error = xfs_sysfs_init(&mp->m_error_kobj, &xfs_error_ktype,
&mp->m_kobj, "error");
if (error)
return error;
goto out_remove_stats_dir;
/* .../xfs/<dev>/error/fail_at_unmount */
error = sysfs_create_file(&mp->m_error_kobj.kobject,
ATTR_LIST(fail_at_unmount));
if (error)
goto out_error;
goto out_remove_error_dir;
/* .../xfs/<dev>/error/metadata/ */
error = xfs_error_sysfs_init_class(mp, XFS_ERR_METADATA,
"metadata", &mp->m_error_meta_kobj,
xfs_error_meta_init);
if (error)
goto out_error;
goto out_remove_error_dir;
if (IS_ENABLED(CONFIG_XFS_RT) && xfs_has_zoned(mp)) {
/* .../xfs/<dev>/zoned/ */
error = xfs_sysfs_init(&mp->m_zoned_kobj, &xfs_zoned_ktype,
&mp->m_kobj, "zoned");
if (error)
goto out_remove_error_dir;
}
return 0;
out_error:
out_remove_error_dir:
xfs_sysfs_del(&mp->m_error_kobj);
out_remove_stats_dir:
xfs_sysfs_del(&mp->m_stats.xs_kobj);
out_remove_fsdir:
xfs_sysfs_del(&mp->m_kobj);
return error;
}
void
xfs_error_sysfs_del(
xfs_mount_sysfs_del(
struct xfs_mount *mp)
{
struct xfs_error_cfg *cfg;
int i, j;
if (IS_ENABLED(CONFIG_XFS_RT) && xfs_has_zoned(mp))
xfs_sysfs_del(&mp->m_zoned_kobj);
for (i = 0; i < XFS_ERR_CLASS_MAX; i++) {
for (j = 0; j < XFS_ERR_ERRNO_MAX; j++) {
cfg = &mp->m_error_cfg[i][j];
@ -749,6 +808,8 @@ xfs_error_sysfs_del(
}
xfs_sysfs_del(&mp->m_error_meta_kobj);
xfs_sysfs_del(&mp->m_error_kobj);
xfs_sysfs_del(&mp->m_stats.xs_kobj);
xfs_sysfs_del(&mp->m_kobj);
}
struct xfs_error_cfg *

5
fs/xfs/xfs_sysfs.h
View File

@ -7,7 +7,6 @@
#ifndef __XFS_SYSFS_H__
#define __XFS_SYSFS_H__
extern const struct kobj_type xfs_mp_ktype; /* xfs_mount */
extern const struct kobj_type xfs_dbg_ktype; /* debug */
extern const struct kobj_type xfs_log_ktype; /* xlog */
extern const struct kobj_type xfs_stats_ktype; /* stats */
@ -53,7 +52,7 @@ xfs_sysfs_del(
wait_for_completion(&kobj->complete);
}
int xfs_error_sysfs_init(struct xfs_mount *mp);
void xfs_error_sysfs_del(struct xfs_mount *mp);
int xfs_mount_sysfs_init(struct xfs_mount *mp);
void xfs_mount_sysfs_del(struct xfs_mount *mp);
#endif /* __XFS_SYSFS_H__ */

2
fs/xfs/xfs_trace.c
View File

@ -49,6 +49,8 @@
#include "xfs_metafile.h"
#include "xfs_metadir.h"
#include "xfs_rtgroup.h"
#include "xfs_zone_alloc.h"
#include "xfs_zone_priv.h"
/*
* We include this last to have the helpers above available for the trace

214
fs/xfs/xfs_trace.h
View File

@ -102,6 +102,7 @@ struct xfs_rmap_intent;
struct xfs_refcount_intent;
struct xfs_metadir_update;
struct xfs_rtgroup;
struct xfs_open_zone;
#define XFS_ATTR_FILTER_FLAGS \
{ XFS_ATTR_ROOT, "ROOT" }, \
@ -265,6 +266,152 @@ DEFINE_GROUP_REF_EVENT(xfs_group_grab);
DEFINE_GROUP_REF_EVENT(xfs_group_grab_next_tag);
DEFINE_GROUP_REF_EVENT(xfs_group_rele);
#ifdef CONFIG_XFS_RT
DECLARE_EVENT_CLASS(xfs_zone_class,
TP_PROTO(struct xfs_rtgroup *rtg),
TP_ARGS(rtg),
TP_STRUCT__entry(
__field(dev_t, dev)
__field(xfs_rgnumber_t, rgno)
__field(xfs_rgblock_t, used)
__field(unsigned int, nr_open)
),
TP_fast_assign(
struct xfs_mount *mp = rtg_mount(rtg);
__entry->dev = mp->m_super->s_dev;
__entry->rgno = rtg_rgno(rtg);
__entry->used = rtg_rmap(rtg)->i_used_blocks;
__entry->nr_open = mp->m_zone_info->zi_nr_open_zones;
),
TP_printk("dev %d:%d rgno 0x%x used 0x%x nr_open %u",
MAJOR(__entry->dev), MINOR(__entry->dev),
__entry->rgno,
__entry->used,
__entry->nr_open)
);
#define DEFINE_ZONE_EVENT(name) \
DEFINE_EVENT(xfs_zone_class, name, \
TP_PROTO(struct xfs_rtgroup *rtg), \
TP_ARGS(rtg))
DEFINE_ZONE_EVENT(xfs_zone_emptied);
DEFINE_ZONE_EVENT(xfs_zone_full);
DEFINE_ZONE_EVENT(xfs_zone_opened);
DEFINE_ZONE_EVENT(xfs_zone_reset);
DEFINE_ZONE_EVENT(xfs_zone_gc_target_opened);
TRACE_EVENT(xfs_zone_free_blocks,
TP_PROTO(struct xfs_rtgroup *rtg, xfs_rgblock_t rgbno,
xfs_extlen_t len),
TP_ARGS(rtg, rgbno, len),
TP_STRUCT__entry(
__field(dev_t, dev)
__field(xfs_rgnumber_t, rgno)
__field(xfs_rgblock_t, used)
__field(xfs_rgblock_t, rgbno)
__field(xfs_extlen_t, len)
),
TP_fast_assign(
__entry->dev = rtg_mount(rtg)->m_super->s_dev;
__entry->rgno = rtg_rgno(rtg);
__entry->used = rtg_rmap(rtg)->i_used_blocks;
__entry->rgbno = rgbno;
__entry->len = len;
),
TP_printk("dev %d:%d rgno 0x%x used 0x%x rgbno 0x%x len 0x%x",
MAJOR(__entry->dev), MINOR(__entry->dev),
__entry->rgno,
__entry->used,
__entry->rgbno,
__entry->len)
);
DECLARE_EVENT_CLASS(xfs_zone_alloc_class,
TP_PROTO(struct xfs_open_zone *oz, xfs_rgblock_t rgbno,
xfs_extlen_t len),
TP_ARGS(oz, rgbno, len),
TP_STRUCT__entry(
__field(dev_t, dev)
__field(xfs_rgnumber_t, rgno)
__field(xfs_rgblock_t, used)
__field(xfs_rgblock_t, written)
__field(xfs_rgblock_t, write_pointer)
__field(xfs_rgblock_t, rgbno)
__field(xfs_extlen_t, len)
),
TP_fast_assign(
__entry->dev = rtg_mount(oz->oz_rtg)->m_super->s_dev;
__entry->rgno = rtg_rgno(oz->oz_rtg);
__entry->used = rtg_rmap(oz->oz_rtg)->i_used_blocks;
__entry->written = oz->oz_written;
__entry->write_pointer = oz->oz_write_pointer;
__entry->rgbno = rgbno;
__entry->len = len;
),
TP_printk("dev %d:%d rgno 0x%x used 0x%x written 0x%x wp 0x%x rgbno 0x%x len 0x%x",
MAJOR(__entry->dev), MINOR(__entry->dev),
__entry->rgno,
__entry->used,
__entry->written,
__entry->write_pointer,
__entry->rgbno,
__entry->len)
);
#define DEFINE_ZONE_ALLOC_EVENT(name) \
DEFINE_EVENT(xfs_zone_alloc_class, name, \
TP_PROTO(struct xfs_open_zone *oz, xfs_rgblock_t rgbno, \
xfs_extlen_t len), \
TP_ARGS(oz, rgbno, len))
DEFINE_ZONE_ALLOC_EVENT(xfs_zone_record_blocks);
DEFINE_ZONE_ALLOC_EVENT(xfs_zone_alloc_blocks);
TRACE_EVENT(xfs_zone_gc_select_victim,
TP_PROTO(struct xfs_rtgroup *rtg, unsigned int bucket),
TP_ARGS(rtg, bucket),
TP_STRUCT__entry(
__field(dev_t, dev)
__field(xfs_rgnumber_t, rgno)
__field(xfs_rgblock_t, used)
__field(unsigned int, bucket)
),
TP_fast_assign(
__entry->dev = rtg_mount(rtg)->m_super->s_dev;
__entry->rgno = rtg_rgno(rtg);
__entry->used = rtg_rmap(rtg)->i_used_blocks;
__entry->bucket = bucket;
),
TP_printk("dev %d:%d rgno 0x%x used 0x%x bucket %u",
MAJOR(__entry->dev), MINOR(__entry->dev),
__entry->rgno,
__entry->used,
__entry->bucket)
);
TRACE_EVENT(xfs_zones_mount,
TP_PROTO(struct xfs_mount *mp),
TP_ARGS(mp),
TP_STRUCT__entry(
__field(dev_t, dev)
__field(xfs_rgnumber_t, rgcount)
__field(uint32_t, blocks)
__field(unsigned int, max_open_zones)
),
TP_fast_assign(
__entry->dev = mp->m_super->s_dev;
__entry->rgcount = mp->m_sb.sb_rgcount;
__entry->blocks = mp->m_groups[XG_TYPE_RTG].blocks;
__entry->max_open_zones = mp->m_max_open_zones;
),
TP_printk("dev %d:%d zoned %u blocks_per_zone %u, max_open %u",
MAJOR(__entry->dev), MINOR(__entry->dev),
__entry->rgcount,
__entry->blocks,
__entry->max_open_zones)
);
#endif /* CONFIG_XFS_RT */
TRACE_EVENT(xfs_inodegc_worker,
TP_PROTO(struct xfs_mount *mp, unsigned int shrinker_hits),
TP_ARGS(mp, shrinker_hits),
@ -1596,6 +1743,7 @@ DEFINE_SIMPLE_IO_EVENT(xfs_end_io_direct_write);
DEFINE_SIMPLE_IO_EVENT(xfs_end_io_direct_write_unwritten);
DEFINE_SIMPLE_IO_EVENT(xfs_end_io_direct_write_append);
DEFINE_SIMPLE_IO_EVENT(xfs_file_splice_read);
DEFINE_SIMPLE_IO_EVENT(xfs_zoned_map_blocks);
DECLARE_EVENT_CLASS(xfs_itrunc_class,
TP_PROTO(struct xfs_inode *ip, xfs_fsize_t new_size),
@ -3983,6 +4131,7 @@ DEFINE_SIMPLE_IO_EVENT(xfs_reflink_cancel_cow_range);
DEFINE_SIMPLE_IO_EVENT(xfs_reflink_end_cow);
DEFINE_INODE_IREC_EVENT(xfs_reflink_cow_remap_from);
DEFINE_INODE_IREC_EVENT(xfs_reflink_cow_remap_to);
DEFINE_INODE_IREC_EVENT(xfs_reflink_cow_remap_skip);
DEFINE_INODE_ERROR_EVENT(xfs_reflink_cancel_cow_range_error);
DEFINE_INODE_ERROR_EVENT(xfs_reflink_end_cow_error);
@ -5606,11 +5755,10 @@ DEFINE_METADIR_EVENT(xfs_metadir_lookup);
/* metadata inode space reservations */
DECLARE_EVENT_CLASS(xfs_metafile_resv_class,
TP_PROTO(struct xfs_inode *ip, xfs_filblks_t len),
TP_ARGS(ip, len),
TP_PROTO(struct xfs_mount *mp, xfs_filblks_t len),
TP_ARGS(mp, len),
TP_STRUCT__entry(
__field(dev_t, dev)
__field(xfs_ino_t, ino)
__field(unsigned long long, freeblks)
__field(unsigned long long, reserved)
__field(unsigned long long, asked)
@ -5618,19 +5766,15 @@ DECLARE_EVENT_CLASS(xfs_metafile_resv_class,
__field(unsigned long long, len)
),
TP_fast_assign(
struct xfs_mount *mp = ip->i_mount;
__entry->dev = mp->m_super->s_dev;
__entry->ino = ip->i_ino;
__entry->freeblks = percpu_counter_sum(&mp->m_fdblocks);
__entry->reserved = ip->i_delayed_blks;
__entry->asked = ip->i_meta_resv_asked;
__entry->used = ip->i_nblocks;
__entry->freeblks = xfs_sum_freecounter_raw(mp, XC_FREE_BLOCKS);
__entry->reserved = mp->m_metafile_resv_avail;
__entry->asked = mp->m_metafile_resv_target;
__entry->used = mp->m_metafile_resv_used;
__entry->len = len;
),
TP_printk("dev %d:%d ino 0x%llx freeblks %llu resv %llu ask %llu used %llu len %llu",
TP_printk("dev %d:%d freeblks %llu resv %llu ask %llu used %llu len %llu",
MAJOR(__entry->dev), MINOR(__entry->dev),
__entry->ino,
__entry->freeblks,
__entry->reserved,
__entry->asked,
@ -5639,14 +5783,14 @@ DECLARE_EVENT_CLASS(xfs_metafile_resv_class,
)
#define DEFINE_METAFILE_RESV_EVENT(name) \
DEFINE_EVENT(xfs_metafile_resv_class, name, \
TP_PROTO(struct xfs_inode *ip, xfs_filblks_t len), \
TP_ARGS(ip, len))
TP_PROTO(struct xfs_mount *mp, xfs_filblks_t len), \
TP_ARGS(mp, len))
DEFINE_METAFILE_RESV_EVENT(xfs_metafile_resv_init);
DEFINE_METAFILE_RESV_EVENT(xfs_metafile_resv_free);
DEFINE_METAFILE_RESV_EVENT(xfs_metafile_resv_alloc_space);
DEFINE_METAFILE_RESV_EVENT(xfs_metafile_resv_free_space);
DEFINE_METAFILE_RESV_EVENT(xfs_metafile_resv_critical);
DEFINE_INODE_ERROR_EVENT(xfs_metafile_resv_init_error);
DEFINE_METAFILE_RESV_EVENT(xfs_metafile_resv_init_error);
#ifdef CONFIG_XFS_RT
TRACE_EVENT(xfs_growfs_check_rtgeom,
@ -5669,6 +5813,46 @@ TRACE_EVENT(xfs_growfs_check_rtgeom,
);
#endif /* CONFIG_XFS_RT */
TRACE_DEFINE_ENUM(XC_FREE_BLOCKS);
TRACE_DEFINE_ENUM(XC_FREE_RTEXTENTS);
TRACE_DEFINE_ENUM(XC_FREE_RTAVAILABLE);
DECLARE_EVENT_CLASS(xfs_freeblocks_resv_class,
TP_PROTO(struct xfs_mount *mp, enum xfs_free_counter ctr,
uint64_t delta, unsigned long caller_ip),
TP_ARGS(mp, ctr, delta, caller_ip),
TP_STRUCT__entry(
__field(dev_t, dev)
__field(enum xfs_free_counter, ctr)
__field(uint64_t, delta)
__field(uint64_t, avail)
__field(uint64_t, total)
__field(unsigned long, caller_ip)
),
TP_fast_assign(
__entry->dev = mp->m_super->s_dev;
__entry->ctr = ctr;
__entry->delta = delta;
__entry->avail = mp->m_free[ctr].res_avail;
__entry->total = mp->m_free[ctr].res_total;
__entry->caller_ip = caller_ip;
),
TP_printk("dev %d:%d ctr %s delta %llu avail %llu total %llu caller %pS",
MAJOR(__entry->dev), MINOR(__entry->dev),
__print_symbolic(__entry->ctr, XFS_FREECOUNTER_STR),
__entry->delta,
__entry->avail,
__entry->total,
(char *)__entry->caller_ip)
)
#define DEFINE_FREEBLOCKS_RESV_EVENT(name) \
DEFINE_EVENT(xfs_freeblocks_resv_class, name, \
TP_PROTO(struct xfs_mount *mp, enum xfs_free_counter ctr, \
uint64_t delta, unsigned long caller_ip), \
TP_ARGS(mp, ctr, delta, caller_ip))
DEFINE_FREEBLOCKS_RESV_EVENT(xfs_freecounter_reserved);
DEFINE_FREEBLOCKS_RESV_EVENT(xfs_freecounter_enospc);
#endif /* _TRACE_XFS_H */
#undef TRACE_INCLUDE_PATH

1220
fs/xfs/xfs_zone_alloc.c Normal file
View File

File diff suppressed because it is too large Load Diff

70
fs/xfs/xfs_zone_alloc.h Normal file
View File

@ -0,0 +1,70 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _XFS_ZONE_ALLOC_H
#define _XFS_ZONE_ALLOC_H
struct iomap_ioend;
struct xfs_open_zone;
struct xfs_zone_alloc_ctx {
struct xfs_open_zone *open_zone;
xfs_filblks_t reserved_blocks;
};
/*
* Grab any available space, even if it is less than what the caller asked for.
*/
#define XFS_ZR_GREEDY (1U << 0)
/*
* Only grab instantly available space, don't wait or GC.
*/
#define XFS_ZR_NOWAIT (1U << 1)
/*
* Dip into the reserved pool.
*/
#define XFS_ZR_RESERVED (1U << 2)
int xfs_zoned_space_reserve(struct xfs_inode *ip, xfs_filblks_t count_fsb,
unsigned int flags, struct xfs_zone_alloc_ctx *ac);
void xfs_zoned_space_unreserve(struct xfs_inode *ip,
struct xfs_zone_alloc_ctx *ac);
void xfs_zoned_add_available(struct xfs_mount *mp, xfs_filblks_t count_fsb);
void xfs_zone_alloc_and_submit(struct iomap_ioend *ioend,
struct xfs_open_zone **oz);
int xfs_zone_free_blocks(struct xfs_trans *tp, struct xfs_rtgroup *rtg,
xfs_fsblock_t fsbno, xfs_filblks_t len);
int xfs_zoned_end_io(struct xfs_inode *ip, xfs_off_t offset, xfs_off_t count,
xfs_daddr_t daddr, struct xfs_open_zone *oz,
xfs_fsblock_t old_startblock);
void xfs_open_zone_put(struct xfs_open_zone *oz);
void xfs_zoned_wake_all(struct xfs_mount *mp);
bool xfs_zone_rgbno_is_valid(struct xfs_rtgroup *rtg, xfs_rgnumber_t rgbno);
void xfs_mark_rtg_boundary(struct iomap_ioend *ioend);
uint64_t xfs_zoned_default_resblks(struct xfs_mount *mp,
enum xfs_free_counter ctr);
void xfs_zoned_show_stats(struct seq_file *m, struct xfs_mount *mp);
#ifdef CONFIG_XFS_RT
int xfs_mount_zones(struct xfs_mount *mp);
void xfs_unmount_zones(struct xfs_mount *mp);
void xfs_zone_gc_start(struct xfs_mount *mp);
void xfs_zone_gc_stop(struct xfs_mount *mp);
#else
static inline int xfs_mount_zones(struct xfs_mount *mp)
{
return -EIO;
}
static inline void xfs_unmount_zones(struct xfs_mount *mp)
{
}
static inline void xfs_zone_gc_start(struct xfs_mount *mp)
{
}
static inline void xfs_zone_gc_stop(struct xfs_mount *mp)
{
}
#endif /* CONFIG_XFS_RT */
#endif /* _XFS_ZONE_ALLOC_H */

1165
fs/xfs/xfs_zone_gc.c Normal file
View File

File diff suppressed because it is too large Load Diff

105
fs/xfs/xfs_zone_info.c Normal file
View File

@ -0,0 +1,105 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (c) 2023-2025 Christoph Hellwig.
* Copyright (c) 2024-2025, Western Digital Corporation or its affiliates.
*/
#include "xfs.h"
#include "xfs_shared.h"
#include "xfs_format.h"
#include "xfs_trans_resv.h"
#include "xfs_mount.h"
#include "xfs_inode.h"
#include "xfs_rtgroup.h"
#include "xfs_zone_alloc.h"
#include "xfs_zone_priv.h"
static const char xfs_write_hint_shorthand[6][16] = {
"NOT_SET", "NONE", "SHORT", "MEDIUM", "LONG", "EXTREME"};
static inline const char *
xfs_write_hint_to_str(
uint8_t write_hint)
{
if (write_hint > WRITE_LIFE_EXTREME)
return "UNKNOWN";
return xfs_write_hint_shorthand[write_hint];
}
static void
xfs_show_open_zone(
struct seq_file *m,
struct xfs_open_zone *oz)
{
seq_printf(m, "\t zone %d, wp %u, written %u, used %u, hint %s\n",
rtg_rgno(oz->oz_rtg),
oz->oz_write_pointer, oz->oz_written,
rtg_rmap(oz->oz_rtg)->i_used_blocks,
xfs_write_hint_to_str(oz->oz_write_hint));
}
static void
xfs_show_full_zone_used_distribution(
struct seq_file *m,
struct xfs_mount *mp)
{
struct xfs_zone_info *zi = mp->m_zone_info;
unsigned int reclaimable = 0, full, i;
spin_lock(&zi->zi_used_buckets_lock);
for (i = 0; i < XFS_ZONE_USED_BUCKETS; i++) {
unsigned int entries = zi->zi_used_bucket_entries[i];
seq_printf(m, "\t %2u..%2u%%: %u\n",
i * (100 / XFS_ZONE_USED_BUCKETS),
(i + 1) * (100 / XFS_ZONE_USED_BUCKETS) - 1,
entries);
reclaimable += entries;
}
spin_unlock(&zi->zi_used_buckets_lock);
full = mp->m_sb.sb_rgcount;
if (zi->zi_open_gc_zone)
full--;
full -= zi->zi_nr_open_zones;
full -= atomic_read(&zi->zi_nr_free_zones);
full -= reclaimable;
seq_printf(m, "\t 100%%: %u\n", full);
}
void
xfs_zoned_show_stats(
struct seq_file *m,
struct xfs_mount *mp)
{
struct xfs_zone_info *zi = mp->m_zone_info;
struct xfs_open_zone *oz;
seq_puts(m, "\n");
seq_printf(m, "\tuser free RT blocks: %lld\n",
xfs_sum_freecounter(mp, XC_FREE_RTEXTENTS));
seq_printf(m, "\treserved free RT blocks: %lld\n",
mp->m_free[XC_FREE_RTEXTENTS].res_avail);
seq_printf(m, "\tuser available RT blocks: %lld\n",
xfs_sum_freecounter(mp, XC_FREE_RTAVAILABLE));
seq_printf(m, "\treserved available RT blocks: %lld\n",
mp->m_free[XC_FREE_RTAVAILABLE].res_avail);
seq_printf(m, "\tRT reservations required: %d\n",
!list_empty_careful(&zi->zi_reclaim_reservations));
seq_printf(m, "\tRT GC required: %d\n",
xfs_zoned_need_gc(mp));
seq_printf(m, "\tfree zones: %d\n", atomic_read(&zi->zi_nr_free_zones));
seq_puts(m, "\topen zones:\n");
spin_lock(&zi->zi_open_zones_lock);
list_for_each_entry(oz, &zi->zi_open_zones, oz_entry)
xfs_show_open_zone(m, oz);
if (zi->zi_open_gc_zone) {
seq_puts(m, "\topen gc zone:\n");
xfs_show_open_zone(m, zi->zi_open_gc_zone);
}
spin_unlock(&zi->zi_open_zones_lock);
seq_puts(m, "\tused blocks distribution (fully written zones):\n");
xfs_show_full_zone_used_distribution(m, mp);
}

119
fs/xfs/xfs_zone_priv.h Normal file
View File

@ -0,0 +1,119 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _XFS_ZONE_PRIV_H
#define _XFS_ZONE_PRIV_H
struct xfs_open_zone {
/*
* Entry in the open zone list and refcount. Protected by
* zi_open_zones_lock in struct xfs_zone_info.
*/
struct list_head oz_entry;
atomic_t oz_ref;
/*
* oz_write_pointer is the write pointer at which space is handed out
* for conventional zones, or simple the count of blocks handed out
* so far for sequential write required zones and is protected by
* oz_alloc_lock/
*/
spinlock_t oz_alloc_lock;
xfs_rgblock_t oz_write_pointer;
/*
* oz_written is the number of blocks for which we've received a
* write completion. oz_written must always be <= oz_write_pointer
* and is protected by the ILOCK of the rmap inode.
*/
xfs_rgblock_t oz_written;
/*
* Write hint (data temperature) assigned to this zone, or
* WRITE_LIFE_NOT_SET if none was set.
*/
enum rw_hint oz_write_hint;
/*
* Is this open zone used for garbage collection? There can only be a
* single open GC zone, which is pointed to by zi_open_gc_zone in
* struct xfs_zone_info. Constant over the life time of an open zone.
*/
bool oz_is_gc;
/*
* Pointer to the RT groups structure for this open zone. Constant over
* the life time of an open zone.
*/
struct xfs_rtgroup *oz_rtg;
};
/*
* Number of bitmap buckets to track reclaimable zones. There are 10 buckets
* so that each 10% of the usable capacity get their own bucket and GC can
* only has to walk the bitmaps of the lesser used zones if there are any.
*/
#define XFS_ZONE_USED_BUCKETS 10u
struct xfs_zone_info {
/*
* List of pending space reservations:
*/
spinlock_t zi_reservation_lock;
struct list_head zi_reclaim_reservations;
/*
* List and number of open zones:
*/
spinlock_t zi_open_zones_lock;
struct list_head zi_open_zones;
unsigned int zi_nr_open_zones;
/*
* Free zone search cursor and number of free zones:
*/
unsigned long zi_free_zone_cursor;
atomic_t zi_nr_free_zones;
/*
* Wait queue to wait for free zones or open zone resources to become
* available:
*/
wait_queue_head_t zi_zone_wait;
/*
* Pointer to the GC thread, and the current open zone used by GC
* (if any).
*
* zi_open_gc_zone is mostly private to the GC thread, but can be read
* for debugging from other threads, in which case zi_open_zones_lock
* must be taken to access it.
*/
struct task_struct *zi_gc_thread;
struct xfs_open_zone *zi_open_gc_zone;
/*
* List of zones that need a reset:
*/
spinlock_t zi_reset_list_lock;
struct xfs_group *zi_reset_list;
/*
* A set of bitmaps to bucket-sort reclaimable zones by used blocks to help
* garbage collection to quickly find the best candidate for reclaim.
*/
spinlock_t zi_used_buckets_lock;
unsigned int zi_used_bucket_entries[XFS_ZONE_USED_BUCKETS];
unsigned long *zi_used_bucket_bitmap[XFS_ZONE_USED_BUCKETS];
};
struct xfs_open_zone *xfs_open_zone(struct xfs_mount *mp,
enum rw_hint write_hint, bool is_gc);
int xfs_zone_gc_reset_sync(struct xfs_rtgroup *rtg);
bool xfs_zoned_need_gc(struct xfs_mount *mp);
int xfs_zone_gc_mount(struct xfs_mount *mp);
void xfs_zone_gc_unmount(struct xfs_mount *mp);
void xfs_zoned_resv_wake_all(struct xfs_mount *mp);
#endif /* _XFS_ZONE_PRIV_H */

263
fs/xfs/xfs_zone_space_resv.c Normal file
View File

@ -0,0 +1,263 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (c) 2023-2025 Christoph Hellwig.
* Copyright (c) 2024-2025, Western Digital Corporation or its affiliates.
*/
#include "xfs.h"
#include "xfs_shared.h"
#include "xfs_format.h"
#include "xfs_trans_resv.h"
#include "xfs_mount.h"
#include "xfs_inode.h"
#include "xfs_rtbitmap.h"
#include "xfs_zone_alloc.h"
#include "xfs_zone_priv.h"
#include "xfs_zones.h"
/*
* Note: the zoned allocator does not support a rtextsize > 1, so this code and
* the allocator itself uses file system blocks interchangeable with realtime
* extents without doing the otherwise required conversions.
*/
/*
* Per-task space reservation.
*
* Tasks that need to wait for GC to free up space allocate one of these
* on-stack and adds it to the per-mount zi_reclaim_reservations lists.
* The GC thread will then wake the tasks in order when space becomes available.
*/
struct xfs_zone_reservation {
struct list_head entry;
struct task_struct *task;
xfs_filblks_t count_fsb;
};
/*
* Calculate the number of reserved blocks.
*
* XC_FREE_RTEXTENTS counts the user available capacity, to which the file
* system can be filled, while XC_FREE_RTAVAILABLE counts the blocks instantly
* available for writes without waiting for GC.
*
* For XC_FREE_RTAVAILABLE only the smaller reservation required for GC and
* block zeroing is excluded from the user capacity, while XC_FREE_RTEXTENTS
* is further restricted by at least one zone as well as the optional
* persistently reserved blocks. This allows the allocator to run more
* smoothly by not always triggering GC.
*/
uint64_t
xfs_zoned_default_resblks(
struct xfs_mount *mp,
enum xfs_free_counter ctr)
{
switch (ctr) {
case XC_FREE_RTEXTENTS:
return (uint64_t)XFS_RESERVED_ZONES *
mp->m_groups[XG_TYPE_RTG].blocks +
mp->m_sb.sb_rtreserved;
case XC_FREE_RTAVAILABLE:
return (uint64_t)XFS_GC_ZONES *
mp->m_groups[XG_TYPE_RTG].blocks;
default:
ASSERT(0);
return 0;
}
}
void
xfs_zoned_resv_wake_all(
struct xfs_mount *mp)
{
struct xfs_zone_info *zi = mp->m_zone_info;
struct xfs_zone_reservation *reservation;
spin_lock(&zi->zi_reservation_lock);
list_for_each_entry(reservation, &zi->zi_reclaim_reservations, entry)
wake_up_process(reservation->task);
spin_unlock(&zi->zi_reservation_lock);
}
void
xfs_zoned_add_available(
struct xfs_mount *mp,
xfs_filblks_t count_fsb)
{
struct xfs_zone_info *zi = mp->m_zone_info;
struct xfs_zone_reservation *reservation;
if (list_empty_careful(&zi->zi_reclaim_reservations)) {
xfs_add_freecounter(mp, XC_FREE_RTAVAILABLE, count_fsb);
return;
}
spin_lock(&zi->zi_reservation_lock);
xfs_add_freecounter(mp, XC_FREE_RTAVAILABLE, count_fsb);
count_fsb = xfs_sum_freecounter(mp, XC_FREE_RTAVAILABLE);
list_for_each_entry(reservation, &zi->zi_reclaim_reservations, entry) {
if (reservation->count_fsb > count_fsb)
break;
wake_up_process(reservation->task);
count_fsb -= reservation->count_fsb;
}
spin_unlock(&zi->zi_reservation_lock);
}
static int
xfs_zoned_space_wait_error(
struct xfs_mount *mp)
{
if (xfs_is_shutdown(mp))
return -EIO;
if (fatal_signal_pending(current))
return -EINTR;
return 0;
}
static int
xfs_zoned_reserve_available(
struct xfs_inode *ip,
xfs_filblks_t count_fsb,
unsigned int flags)
{
struct xfs_mount *mp = ip->i_mount;
struct xfs_zone_info *zi = mp->m_zone_info;
struct xfs_zone_reservation reservation = {
.task = current,
.count_fsb = count_fsb,
};
int error;
/*
* If there are no waiters, try to directly grab the available blocks
* from the percpu counter.
*
* If the caller wants to dip into the reserved pool also bypass the
* wait list. This relies on the fact that we have a very graciously
* sized reserved pool that always has enough space. If the reserved
* allocations fail we're in trouble.
*/
if (likely(list_empty_careful(&zi->zi_reclaim_reservations) ||
(flags & XFS_ZR_RESERVED))) {
error = xfs_dec_freecounter(mp, XC_FREE_RTAVAILABLE, count_fsb,
flags & XFS_ZR_RESERVED);
if (error != -ENOSPC)
return error;
}
if (flags & XFS_ZR_NOWAIT)
return -EAGAIN;
spin_lock(&zi->zi_reservation_lock);
list_add_tail(&reservation.entry, &zi->zi_reclaim_reservations);
while ((error = xfs_zoned_space_wait_error(mp)) == 0) {
set_current_state(TASK_KILLABLE);
error = xfs_dec_freecounter(mp, XC_FREE_RTAVAILABLE, count_fsb,
flags & XFS_ZR_RESERVED);
if (error != -ENOSPC)
break;
/*
* Make sure to start GC if it is not running already. As we
* check the rtavailable count when filling up zones, GC is
* normally already running at this point, but in some setups
* with very few zones we may completely run out of non-
* reserved blocks in between filling zones.
*/
if (!xfs_is_zonegc_running(mp))
wake_up_process(zi->zi_gc_thread);
/*
* If there is no reclaimable group left and we aren't still
* processing a pending GC request give up as we're fully out
* of space.
*/
if (!xfs_group_marked(mp, XG_TYPE_RTG, XFS_RTG_RECLAIMABLE) &&
!xfs_is_zonegc_running(mp))
break;
spin_unlock(&zi->zi_reservation_lock);
schedule();
spin_lock(&zi->zi_reservation_lock);
}
list_del(&reservation.entry);
spin_unlock(&zi->zi_reservation_lock);
__set_current_state(TASK_RUNNING);
return error;
}
/*
* Implement greedy space allocation for short writes by trying to grab all
* that is left after locking out other threads from trying to do the same.
*
* This isn't exactly optimal and can hopefully be replaced by a proper
* percpu_counter primitive one day.
*/
static int
xfs_zoned_reserve_extents_greedy(
struct xfs_inode *ip,
xfs_filblks_t *count_fsb,
unsigned int flags)
{
struct xfs_mount *mp = ip->i_mount;
struct xfs_zone_info *zi = mp->m_zone_info;
s64 len = *count_fsb;
int error = -ENOSPC;
spin_lock(&zi->zi_reservation_lock);
len = min(len, xfs_sum_freecounter(mp, XC_FREE_RTEXTENTS));
if (len > 0) {
*count_fsb = len;
error = xfs_dec_freecounter(mp, XC_FREE_RTEXTENTS, *count_fsb,
flags & XFS_ZR_RESERVED);
}
spin_unlock(&zi->zi_reservation_lock);
return error;
}
int
xfs_zoned_space_reserve(
struct xfs_inode *ip,
xfs_filblks_t count_fsb,
unsigned int flags,
struct xfs_zone_alloc_ctx *ac)
{
struct xfs_mount *mp = ip->i_mount;
int error;
ASSERT(ac->reserved_blocks == 0);
ASSERT(ac->open_zone == NULL);
error = xfs_dec_freecounter(mp, XC_FREE_RTEXTENTS, count_fsb,
flags & XFS_ZR_RESERVED);
if (error == -ENOSPC && (flags & XFS_ZR_GREEDY) && count_fsb > 1)
error = xfs_zoned_reserve_extents_greedy(ip, &count_fsb, flags);
if (error)
return error;
error = xfs_zoned_reserve_available(ip, count_fsb, flags);
if (error) {
xfs_add_freecounter(mp, XC_FREE_RTEXTENTS, count_fsb);
return error;
}
ac->reserved_blocks = count_fsb;
return 0;
}
void
xfs_zoned_space_unreserve(
struct xfs_inode *ip,
struct xfs_zone_alloc_ctx *ac)
{
if (ac->reserved_blocks > 0) {
struct xfs_mount *mp = ip->i_mount;
xfs_zoned_add_available(mp, ac->reserved_blocks);
xfs_add_freecounter(mp, XC_FREE_RTEXTENTS, ac->reserved_blocks);
}
if (ac->open_zone)
xfs_open_zone_put(ac->open_zone);
}

2
fs/zonefs/file.c
View File

@ -299,7 +299,7 @@ static vm_fault_t zonefs_filemap_page_mkwrite(struct vm_fault *vmf)
/* Serialize against truncates */
filemap_invalidate_lock_shared(inode->i_mapping);
ret = iomap_page_mkwrite(vmf, &zonefs_write_iomap_ops);
ret = iomap_page_mkwrite(vmf, &zonefs_write_iomap_ops, NULL);
filemap_invalidate_unlock_shared(inode->i_mapping);
sb_end_pagefault(inode->i_sb);

110
include/linux/iomap.h
View File

@ -56,6 +56,10 @@ struct vm_fault;
*
* IOMAP_F_BOUNDARY indicates that I/O and I/O completions for this iomap must
* never be merged with the mapping before it.
*
* IOMAP_F_ANON_WRITE indicates that (write) I/O does not have a target block
* assigned to it yet and the file system will do that in the bio submission
* handler, splitting the I/O as needed.
*/
#define IOMAP_F_NEW (1U << 0)
#define IOMAP_F_DIRTY (1U << 1)
@ -68,6 +72,7 @@ struct vm_fault;
#endif /* CONFIG_BUFFER_HEAD */
#define IOMAP_F_XATTR (1U << 5)
#define IOMAP_F_BOUNDARY (1U << 6)
#define IOMAP_F_ANON_WRITE (1U << 7)
/*
* Flags set by the core iomap code during operations:
@ -111,6 +116,8 @@ struct iomap {
static inline sector_t iomap_sector(const struct iomap *iomap, loff_t pos)
{
if (iomap->flags & IOMAP_F_ANON_WRITE)
return U64_MAX; /* invalid */
return (iomap->addr + pos - iomap->offset) >> SECTOR_SHIFT;
}
@ -183,6 +190,7 @@ struct iomap_folio_ops {
#define IOMAP_DAX 0
#endif /* CONFIG_FS_DAX */
#define IOMAP_ATOMIC (1 << 9)
#define IOMAP_DONTCACHE (1 << 10)
struct iomap_ops {
/*
@ -211,8 +219,10 @@ struct iomap_ops {
* calls to iomap_iter(). Treat as read-only in the body.
* @len: The remaining length of the file segment we're operating on.
* It is updated at the same time as @pos.
* @processed: The number of bytes processed by the body in the most recent
* iteration, or a negative errno. 0 causes the iteration to stop.
* @iter_start_pos: The original start pos for the current iomap. Used for
* incremental iter advance.
* @status: Status of the most recent iteration. Zero on success or a negative
* errno on error.
* @flags: Zero or more of the iomap_begin flags above.
* @iomap: Map describing the I/O iteration
* @srcmap: Source map for COW operations
@ -221,7 +231,8 @@ struct iomap_iter {
struct inode *inode;
loff_t pos;
u64 len;
s64 processed;
loff_t iter_start_pos;
int status;
unsigned flags;
struct iomap iomap;
struct iomap srcmap;
@ -229,6 +240,26 @@ struct iomap_iter {
};
int iomap_iter(struct iomap_iter *iter, const struct iomap_ops *ops);
int iomap_iter_advance(struct iomap_iter *iter, u64 *count);
/**
* iomap_length_trim - trimmed length of the current iomap iteration
* @iter: iteration structure
* @pos: File position to trim from.
* @len: Length of the mapping to trim to.
*
* Returns a trimmed length that the operation applies to for the current
* iteration.
*/
static inline u64 iomap_length_trim(const struct iomap_iter *iter, loff_t pos,
u64 len)
{
u64 end = iter->iomap.offset + iter->iomap.length;
if (iter->srcmap.type != IOMAP_HOLE)
end = min(end, iter->srcmap.offset + iter->srcmap.length);
return min(len, end - pos);
}
/**
* iomap_length - length of the current iomap iteration
@ -238,11 +269,17 @@ int iomap_iter(struct iomap_iter *iter, const struct iomap_ops *ops);
*/
static inline u64 iomap_length(const struct iomap_iter *iter)
{
u64 end = iter->iomap.offset + iter->iomap.length;
return iomap_length_trim(iter, iter->pos, iter->len);
}
if (iter->srcmap.type != IOMAP_HOLE)
end = min(end, iter->srcmap.offset + iter->srcmap.length);
return min(iter->len, end - iter->pos);
/**
* iomap_iter_advance_full - advance by the full length of current map
*/
static inline int iomap_iter_advance_full(struct iomap_iter *iter)
{
u64 length = iomap_length(iter);
return iomap_iter_advance(iter, &length);
}
/**
@ -306,12 +343,11 @@ bool iomap_dirty_folio(struct address_space *mapping, struct folio *folio);
int iomap_file_unshare(struct inode *inode, loff_t pos, loff_t len,
const struct iomap_ops *ops);
int iomap_zero_range(struct inode *inode, loff_t pos, loff_t len,
bool *did_zero, const struct iomap_ops *ops);
bool *did_zero, const struct iomap_ops *ops, void *private);
int iomap_truncate_page(struct inode *inode, loff_t pos, bool *did_zero,
const struct iomap_ops *ops);
vm_fault_t iomap_page_mkwrite(struct vm_fault *vmf,
const struct iomap_ops *ops);
const struct iomap_ops *ops, void *private);
vm_fault_t iomap_page_mkwrite(struct vm_fault *vmf, const struct iomap_ops *ops,
void *private);
typedef void (*iomap_punch_t)(struct inode *inode, loff_t offset, loff_t length,
struct iomap *iomap);
void iomap_write_delalloc_release(struct inode *inode, loff_t start_byte,
@ -327,17 +363,43 @@ loff_t iomap_seek_data(struct inode *inode, loff_t offset,
sector_t iomap_bmap(struct address_space *mapping, sector_t bno,
const struct iomap_ops *ops);
/*
* Flags for iomap_ioend->io_flags.
*/
/* shared COW extent */
#define IOMAP_IOEND_SHARED (1U << 0)
/* unwritten extent */
#define IOMAP_IOEND_UNWRITTEN (1U << 1)
/* don't merge into previous ioend */
#define IOMAP_IOEND_BOUNDARY (1U << 2)
/* is direct I/O */
#define IOMAP_IOEND_DIRECT (1U << 3)
/*
* Flags that if set on either ioend prevent the merge of two ioends.
* (IOMAP_IOEND_BOUNDARY also prevents merges, but only one-way)
*/
#define IOMAP_IOEND_NOMERGE_FLAGS \
(IOMAP_IOEND_SHARED | IOMAP_IOEND_UNWRITTEN | IOMAP_IOEND_DIRECT)
/*
* Structure for writeback I/O completions.
*
* File systems implementing ->submit_ioend (for buffered I/O) or ->submit_io
* for direct I/O) can split a bio generated by iomap. In that case the parent
* ioend it was split from is recorded in ioend->io_parent.
*/
struct iomap_ioend {
struct list_head io_list; /* next ioend in chain */
u16 io_type;
u16 io_flags; /* IOMAP_F_* */
u16 io_flags; /* IOMAP_IOEND_* */
struct inode *io_inode; /* file being written to */
size_t io_size; /* size of data within eof */
size_t io_size; /* size of the extent */
atomic_t io_remaining; /* completetion defer count */
int io_error; /* stashed away status */
struct iomap_ioend *io_parent; /* parent for completions */
loff_t io_offset; /* offset in the file */
sector_t io_sector; /* start sector of ioend */
void *io_private; /* file system private data */
struct bio io_bio; /* MUST BE LAST! */
};
@ -362,12 +424,14 @@ struct iomap_writeback_ops {
loff_t offset, unsigned len);
/*
* Optional, allows the file systems to perform actions just before
* submitting the bio and/or override the bio end_io handler for complex
* operations like copy on write extent manipulation or unwritten extent
* conversions.
* Optional, allows the file systems to hook into bio submission,
* including overriding the bi_end_io handler.
*
* Returns 0 if the bio was successfully submitted, or a negative
* error code if status was non-zero or another error happened and
* the bio could not be submitted.
*/
int (*prepare_ioend)(struct iomap_ioend *ioend, int status);
int (*submit_ioend)(struct iomap_writepage_ctx *wpc, int status);
/*
* Optional, allows the file system to discard state on a page where
@ -383,6 +447,10 @@ struct iomap_writepage_ctx {
u32 nr_folios; /* folios added to the ioend */
};
struct iomap_ioend *iomap_init_ioend(struct inode *inode, struct bio *bio,
loff_t file_offset, u16 ioend_flags);
struct iomap_ioend *iomap_split_ioend(struct iomap_ioend *ioend,
unsigned int max_len, bool is_append);
void iomap_finish_ioends(struct iomap_ioend *ioend, int error);
void iomap_ioend_try_merge(struct iomap_ioend *ioend,
struct list_head *more_ioends);
@ -454,4 +522,6 @@ int iomap_swapfile_activate(struct swap_info_struct *sis,
# define iomap_swapfile_activate(sis, swapfile, pagespan, ops) (-EIO)
#endif /* CONFIG_SWAP */
extern struct bio_set iomap_ioend_bioset;
#endif /* LINUX_IOMAP_H */