diff --git a/Documentation/filesystems/erofs.rst b/Documentation/filesystems/erofs.rst index 4230884fb359..d301d9ac946a 100644 --- a/Documentation/filesystems/erofs.rst +++ b/Documentation/filesystems/erofs.rst @@ -137,8 +137,25 @@ fsoffset=%llu Specify block-aligned filesystem offset for the primary d inode_share Enable inode page sharing for this filesystem. Inodes with identical content within the same domain ID can share the page cache. +source=%s (For file-backed mounts) Specify the backing image as a path + or as an already-opened file descriptor. =================== ========================================================= +File-backed mounts +================== + +When CONFIG_EROFS_FS_BACKED_BY_FILE is enabled, EROFS file-backed images +can be mounted directly without a loopback block device. The backing file +can be given either as a path, or as an already-opened file descriptor. + +When a file descriptor is used, the kernel resolves its path and records it +so that /proc/mounts and similar interfaces can still report the mount +source. + +Only regular files are accepted as backing files; to mount an image that +resides on a block device, use the traditional block device mount path +instead. + Sysfs Entries ============= diff --git a/fs/erofs/data.c b/fs/erofs/data.c index d2f01245ee79..20962f8b60a8 100644 --- a/fs/erofs/data.c +++ b/fs/erofs/data.c @@ -30,20 +30,6 @@ void *erofs_bread(struct erofs_buf *buf, erofs_off_t offset, bool need_kmap) { pgoff_t index = (buf->off + offset) >> PAGE_SHIFT; struct folio *folio = NULL; - loff_t fpos; - int err; - - /* - * Metadata access for file-backed mounts reuses page cache of backing - * fs inodes (only folio data will be needed) to prevent double caching. - * However, the data access range must be verified here in advance. - */ - if (buf->file) { - fpos = (loff_t)index << PAGE_SHIFT; - err = rw_verify_area(READ, buf->file, &fpos, PAGE_SIZE); - if (err < 0) - return ERR_PTR(err); - } if (buf->page) { folio = page_folio(buf->page); @@ -52,7 +38,8 @@ void *erofs_bread(struct erofs_buf *buf, erofs_off_t offset, bool need_kmap) } if (!folio || !folio_contains(folio, index)) { erofs_put_metabuf(buf); - folio = read_mapping_folio(buf->mapping, index, buf->file); + folio = read_cache_folio(buf->mapping, index, + buf->mc ? erofs_read_meta_folio : NULL, NULL); if (IS_ERR(folio)) return folio; } @@ -69,19 +56,20 @@ int erofs_init_metabuf(struct erofs_buf *buf, struct super_block *sb, { struct erofs_sb_info *sbi = EROFS_SB(sb); - buf->file = NULL; + buf->mc = false; if (in_metabox) { if (unlikely(!sbi->metabox_inode)) return -EFSCORRUPTED; buf->mapping = sbi->metabox_inode->i_mapping; return 0; } - buf->off = sbi->dif0.fsoff; if (erofs_is_fileio_mode(sbi)) { - buf->file = sbi->dif0.file; /* some fs like FUSE needs it */ - buf->mapping = buf->file->f_mapping; - } else + buf->mapping = sbi->managed_cache->i_mapping; + buf->mc = true; + } else { + buf->off = sbi->dif0.fsoff; buf->mapping = sb->s_bdev->bd_mapping; + } return 0; } diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c index 3c54e95964c9..27caf4bebddc 100644 --- a/fs/erofs/decompressor.c +++ b/fs/erofs/decompressor.c @@ -299,6 +299,7 @@ static const char *z_erofs_transform_plain(struct z_erofs_decompress_req *rq, return ERR_PTR(-EOPNOTSUPP); if (rq->alg == Z_EROFS_COMPRESSION_INTERLACED) { cur = bs - (rq->pageofs_out & (bs - 1)); + DBG_BUGON(rq->pageofs_in & (bs - 1)); pi = (rq->pageofs_in + rq->inputsize - cur) & ~PAGE_MASK; cur = min(cur, rq->outputsize); if (cur && rq->out[0]) { diff --git a/fs/erofs/fileio.c b/fs/erofs/fileio.c index 98cdaa1cd1a7..ebb81a7ffd4c 100644 --- a/fs/erofs/fileio.c +++ b/fs/erofs/fileio.c @@ -194,3 +194,24 @@ const struct address_space_operations erofs_fileio_aops = { .read_folio = erofs_fileio_read_folio, .readahead = erofs_fileio_readahead, }; + +int erofs_read_meta_folio(struct file *file, struct folio *folio) +{ + struct erofs_fileio io = { + .dev = { .m_pa = folio_pos(folio), }, + }; + struct inode *inode = folio_inode(folio); + int err; + + err = erofs_map_dev(inode->i_sb, &io.dev); + if (err) + return err; + + io.rq = erofs_fileio_rq_alloc(&io.dev); + io.rq->bio.bi_iter.bi_sector = + (io.dev.m_dif->fsoff + io.dev.m_pa) >> 9; + erofs_onlinefolio_init(folio); + bio_add_folio_nofail(&io.rq->bio, folio, folio_size(folio), 0); + erofs_fileio_rq_submit(io.rq); + return 0; +} diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h index 57bd21859c65..9b7370f0f3df 100644 --- a/fs/erofs/internal.h +++ b/fs/erofs/internal.h @@ -98,11 +98,9 @@ struct erofs_sb_info { unsigned int sync_decompress; /* strategy for sync decompression */ unsigned int shrinker_run_no; - /* pseudo inode to manage cached pages */ - struct inode *managed_cache; - struct erofs_sb_lz4_info lz4; #endif /* CONFIG_EROFS_FS_ZIP */ + struct inode *managed_cache; /* pseudo inode to cache physical data */ struct inode *packed_inode; struct inode *metabox_inode; struct erofs_dev_context *devs; @@ -176,10 +174,10 @@ enum { struct erofs_buf { struct address_space *mapping; - struct file *file; u64 off; struct page *page; void *base; + bool mc; }; #define __EROFS_BUF_INITIALIZER ((struct erofs_buf){ .page = NULL }) @@ -269,7 +267,7 @@ struct erofs_inode { #ifdef CONFIG_EROFS_FS_ZIP struct { unsigned short z_advise; - unsigned char z_algorithmtype[2]; + unsigned char z_algofmt[2]; unsigned char z_lclusterbits; union { u64 z_tailextent_headlcn; @@ -399,6 +397,12 @@ extern const struct file_operations erofs_ishare_fops; extern const struct iomap_ops z_erofs_iomap_report_ops; +int erofs_setup_managed_cache(struct super_block *sb); +#ifdef CONFIG_EROFS_FS_BACKED_BY_FILE +int erofs_read_meta_folio(struct file *file, struct folio *folio); +#else +#define erofs_read_meta_folio NULL +#endif void *erofs_read_metadata(struct super_block *sb, struct erofs_buf *buf, erofs_off_t *offset, int *lengthp); void erofs_unmap_metabuf(struct erofs_buf *buf); diff --git a/fs/erofs/super.c b/fs/erofs/super.c index 770808685934..0967aace9f7f 100644 --- a/fs/erofs/super.c +++ b/fs/erofs/super.c @@ -386,6 +386,7 @@ static void erofs_default_options(struct erofs_sb_info *sbi) enum { Opt_user_xattr, Opt_acl, Opt_cache_strategy, Opt_dax, Opt_dax_enum, Opt_device, Opt_domain_id, Opt_directio, Opt_fsoffset, Opt_inode_share, + Opt_source, }; static const struct constant_table erofs_param_cache_strategy[] = { @@ -402,17 +403,18 @@ static const struct constant_table erofs_dax_param_enums[] = { }; static const struct fs_parameter_spec erofs_fs_parameters[] = { - fsparam_flag_no("user_xattr", Opt_user_xattr), - fsparam_flag_no("acl", Opt_acl), - fsparam_enum("cache_strategy", Opt_cache_strategy, + fsparam_flag_no("user_xattr", Opt_user_xattr), + fsparam_flag_no("acl", Opt_acl), + fsparam_enum("cache_strategy", Opt_cache_strategy, erofs_param_cache_strategy), - fsparam_flag("dax", Opt_dax), - fsparam_enum("dax", Opt_dax_enum, erofs_dax_param_enums), - fsparam_string("device", Opt_device), - fsparam_string("domain_id", Opt_domain_id), - fsparam_flag_no("directio", Opt_directio), - fsparam_u64("fsoffset", Opt_fsoffset), - fsparam_flag("inode_share", Opt_inode_share), + fsparam_flag("dax", Opt_dax), + fsparam_enum("dax", Opt_dax_enum, erofs_dax_param_enums), + fsparam_string("device", Opt_device), + fsparam_string("domain_id", Opt_domain_id), + fsparam_flag_no("directio", Opt_directio), + fsparam_u64("fsoffset", Opt_fsoffset), + fsparam_flag("inode_share", Opt_inode_share), + fsparam_file_or_string("source", Opt_source), {} }; @@ -437,6 +439,40 @@ static bool erofs_fc_set_dax_mode(struct fs_context *fc, unsigned int mode) return false; } +static int erofs_fc_parse_source(struct fs_context *fc, + struct fs_parameter *param) +{ + struct erofs_sb_info *sbi = fc->s_fs_info; + + if (fc->source || sbi->dif0.file) + return invalf(fc, "Multiple sources"); + + switch (param->type) { + case fs_value_is_string: + fc->source = param->string; + param->string = NULL; + return 0; + case fs_value_is_file: { + char *buf __free(kfree) = kmalloc(PATH_MAX, GFP_KERNEL); + char *p; + + if (!buf) + return -ENOMEM; + p = file_path(param->file, buf, PATH_MAX); + if (IS_ERR(p)) + return PTR_ERR(p); + fc->source = kstrdup(p, GFP_KERNEL); + if (!fc->source) + return -ENOMEM; + sbi->dif0.file = no_free_ptr(param->file); + return 0; + } + default: + WARN_ON_ONCE(true); + return -EINVAL; + } +} + static int erofs_fc_parse_param(struct fs_context *fc, struct fs_parameter *param) { @@ -524,6 +560,8 @@ static int erofs_fc_parse_param(struct fs_context *fc, else set_opt(&sbi->opt, INODE_SHARE); break; + case Opt_source: + return erofs_fc_parse_source(fc, param); } return 0; } @@ -595,6 +633,21 @@ static const struct export_operations erofs_export_ops = { .get_parent = erofs_get_parent, }; +int erofs_setup_managed_cache(struct super_block *sb) +{ + if (!EROFS_SB(sb)->managed_cache) { + struct inode *inode = new_inode(sb); + + if (!inode) + return -ENOMEM; + set_nlink(inode, 1); + inode->i_size = OFFSET_MAX; + mapping_set_gfp_mask(inode->i_mapping, GFP_KERNEL); + EROFS_SB(sb)->managed_cache = inode; + } + return 0; +} + static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) { struct inode *inode; @@ -607,16 +660,16 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) sb->s_op = &erofs_sops; if (!sbi->domain_id && test_opt(&sbi->opt, INODE_SHARE)) { - errorfc(fc, "domain_id is needed when inode_ishare is on"); + errorfc(fc, "domain_id is needed when inode_share is on"); return -EINVAL; } if (test_opt(&sbi->opt, DAX_ALWAYS) && test_opt(&sbi->opt, INODE_SHARE)) { - errorfc(fc, "FSDAX is not allowed when inode_ishare is on"); + errorfc(fc, "FSDAX is not allowed when inode_share is on"); return -EINVAL; } sbi->blkszbits = PAGE_SHIFT; - if (!sb->s_bdev) { + if (erofs_is_fileio_mode(sbi)) { /* * (File-backed mounts) EROFS claims it's safe to nest other * fs contexts (including its own) due to self-controlled RO @@ -631,19 +684,19 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) * It MUST change if another fs plans to support them, which * may also require adjusting FILESYSTEM_MAX_STACK_DEPTH. */ - if (erofs_is_fileio_mode(sbi)) { - inode = file_inode(sbi->dif0.file); - if ((inode->i_sb->s_op == &erofs_sops && - !inode->i_sb->s_bdev) || - inode->i_sb->s_stack_depth) { - erofs_err(sb, "file-backed mounts cannot be applied to stacked fses"); - return -ENOTBLK; - } + inode = file_inode(sbi->dif0.file); + if ((inode->i_sb->s_op == &erofs_sops && + !inode->i_sb->s_bdev) || inode->i_sb->s_stack_depth) { + erofs_err(sb, "file-backed mounts cannot be applied to stacked fses"); + return -ENOTBLK; } sb->s_blocksize = PAGE_SIZE; sb->s_blocksize_bits = PAGE_SHIFT; err = super_setup_bdi(sb); + if (err) + return err; + err = erofs_setup_managed_cache(sb); if (err) return err; @@ -743,13 +796,26 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) static int erofs_fc_get_tree(struct fs_context *fc) { + struct erofs_sb_info *sbi = fc->s_fs_info; int ret; + if (sbi->dif0.file) { + if (!IS_ENABLED(CONFIG_EROFS_FS_BACKED_BY_FILE)) { + errorfc(fc, "source fd option not supported"); + return -EINVAL; + } + if (!S_ISREG(file_inode(sbi->dif0.file)->i_mode) || + !sbi->dif0.file->f_mapping->a_ops->read_folio) { + errorfc(fc, "source is unsupported"); + return -EINVAL; + } + return get_tree_nodev(fc, erofs_fc_fill_super); + } + ret = get_tree_bdev_flags(fc, erofs_fc_fill_super, IS_ENABLED(CONFIG_EROFS_FS_BACKED_BY_FILE) ? GET_TREE_BDEV_QUIET_LOOKUP : 0); if (IS_ENABLED(CONFIG_EROFS_FS_BACKED_BY_FILE) && ret == -ENOTBLK) { - struct erofs_sb_info *sbi = fc->s_fs_info; struct file *file; if (!fc->source) @@ -873,10 +939,8 @@ static void erofs_drop_internal_inodes(struct erofs_sb_info *sbi) sbi->packed_inode = NULL; iput(sbi->metabox_inode); sbi->metabox_inode = NULL; -#ifdef CONFIG_EROFS_FS_ZIP iput(sbi->managed_cache); sbi->managed_cache = NULL; -#endif } static void erofs_kill_sb(struct super_block *sb) diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c index d022d1dff5a1..0de7d561f189 100644 --- a/fs/erofs/zdata.c +++ b/fs/erofs/zdata.c @@ -128,7 +128,17 @@ struct z_erofs_pcluster_slab { #define _PCLP(n) { .maxpages = n } static struct z_erofs_pcluster_slab pcluster_pool[] __read_mostly = { - _PCLP(1), _PCLP(4), _PCLP(16), _PCLP(64), _PCLP(128), + _PCLP(1), + _PCLP(4), +#if Z_EROFS_PCLUSTER_MAX_PAGES > 16 + _PCLP(16), +#endif +#if Z_EROFS_PCLUSTER_MAX_PAGES > 64 + _PCLP(64), +#endif +#if Z_EROFS_PCLUSTER_MAX_PAGES > 128 + _PCLP(128), +#endif _PCLP(Z_EROFS_PCLUSTER_MAX_PAGES + 1) }; @@ -666,21 +676,16 @@ static const struct address_space_operations z_erofs_cache_aops = { int z_erofs_init_super(struct super_block *sb) { - struct inode *inode; int err; err = z_erofs_init_pcpu_workers(sb); if (err) return err; - inode = new_inode(sb); - if (!inode) - return -ENOMEM; - set_nlink(inode, 1); - inode->i_size = OFFSET_MAX; - inode->i_mapping->a_ops = &z_erofs_cache_aops; - mapping_set_gfp_mask(inode->i_mapping, GFP_KERNEL); - EROFS_SB(sb)->managed_cache = inode; + err = erofs_setup_managed_cache(sb); + if (err) + return err; + EROFS_SB(sb)->managed_cache->i_mapping->a_ops = &z_erofs_cache_aops; xa_init(&EROFS_SB(sb)->managed_pslots); return 0; } diff --git a/fs/erofs/zmap.c b/fs/erofs/zmap.c index 5f33af3fdf97..f8981fc74246 100644 --- a/fs/erofs/zmap.c +++ b/fs/erofs/zmap.c @@ -483,14 +483,14 @@ static int z_erofs_map_blocks_fo(struct inode *inode, } if (m.headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN) { - if (vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) + if ((vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) && + !(map->m_flags & EROFS_MAP_META)) map->m_algorithmformat = Z_EROFS_COMPRESSION_INTERLACED; else map->m_algorithmformat = Z_EROFS_COMPRESSION_SHIFTED; - } else if (m.headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) { - map->m_algorithmformat = vi->z_algorithmtype[1]; } else { - map->m_algorithmformat = vi->z_algorithmtype[0]; + map->m_algorithmformat = + vi->z_algofmt[m.headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2]; } if ((flags & EROFS_GET_BLOCKS_FIEMAP) || @@ -604,9 +604,14 @@ static int z_erofs_map_blocks_ext(struct inode *inode, if (map->m_plen & Z_EROFS_EXTENT_PLEN_PARTIAL) map->m_flags |= EROFS_MAP_PARTIAL_REF; map->m_plen &= Z_EROFS_EXTENT_PLEN_MASK; - if (fmt) - map->m_algorithmformat = fmt - 1; - else if (interlaced && !((map->m_pa | map->m_plen) & bmask)) + if (fmt) { + map->m_algorithmformat = --fmt; + if (fmt >= Z_EROFS_COMPRESSION_MAX) { + erofs_err(sb, "unknown algorithm %d @ pos %llu for nid %llu, please upgrade kernel", + fmt, map->m_la, vi->nid); + return -EOPNOTSUPP; + } + } else if (interlaced && !((map->m_pa | map->m_plen) & bmask)) map->m_algorithmformat = Z_EROFS_COMPRESSION_INTERLACED; else @@ -624,7 +629,7 @@ static int z_erofs_fill_inode(struct inode *inode, struct erofs_map_blocks *map) struct super_block *const sb = inode->i_sb; struct z_erofs_map_header *h; erofs_off_t pos; - int err = 0; + int err = 0, nr; if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) { /* @@ -667,12 +672,19 @@ static int z_erofs_fill_inode(struct inode *inode, struct erofs_map_blocks *map) goto done; } - vi->z_algorithmtype[0] = h->h_algorithmtype & 15; - vi->z_algorithmtype[1] = h->h_algorithmtype >> 4; if (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) vi->z_fragmentoff = le32_to_cpu(h->h_fragmentoff); else if (vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER) vi->z_idata_size = le16_to_cpu(h->h_idata_size); + for (nr = 0; nr < 2; ++nr) { + vi->z_algofmt[nr] = (h->h_algorithmtype >> (4 * nr)) & 15; + if (vi->z_algofmt[nr] >= Z_EROFS_COMPRESSION_MAX) { + erofs_err(sb, "unknown HEAD%u format %u for nid %llu, please upgrade kernel", + nr + 1, vi->z_algofmt[nr], vi->nid); + err = -EOPNOTSUPP; + goto out_unlock; + } + } if (!erofs_sb_has_big_pcluster(EROFS_SB(sb)) && vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | @@ -720,12 +732,8 @@ static int z_erofs_map_sanity_check(struct inode *inode, if (!(map->m_flags & EROFS_MAP_MAPPED)) return 0; - if (unlikely(map->m_algorithmformat >= Z_EROFS_COMPRESSION_RUNTIME_MAX)) { - erofs_err(inode->i_sb, "unknown algorithm %d @ pos %llu for nid %llu, please upgrade kernel", - map->m_algorithmformat, map->m_la, EROFS_I(inode)->nid); - return -EOPNOTSUPP; - } + DBG_BUGON(map->m_algorithmformat >= Z_EROFS_COMPRESSION_RUNTIME_MAX); if (map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX) { if (!(sbi->available_compr_algs & BIT(map->m_algorithmformat))) { erofs_err(inode->i_sb, "inconsistent algorithmtype %u for nid %llu",