diff --git a/drivers/block/zram/zcomp.c b/drivers/block/zram/zcomp.c index 974c4691887e..028e2f0f587e 100644 --- a/drivers/block/zram/zcomp.c +++ b/drivers/block/zram/zcomp.c @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include @@ -158,17 +160,32 @@ int zcomp_compress(struct zcomp *comp, struct zcomp_strm *zstrm, } int zcomp_decompress(struct zcomp *comp, struct zcomp_strm *zstrm, - const void *src, unsigned int src_len, void *dst) + struct scatterlist *sg, unsigned int src_len, void *dst) { struct zcomp_req req = { - .src = src, .dst = dst, .src_len = src_len, .dst_len = PAGE_SIZE, }; + void *src = NULL; + int ret; might_sleep(); - return comp->ops->decompress(comp->params, &zstrm->ctx, &req); + + if (sg_is_last(sg)) { + /* the object is contained within one page, read it in-place */ + src = kmap_local_page(sg_page(sg)); + req.src = src + sg->offset; + } else { + /* the object spans two pages, linearize it into local copy */ + sg_copy_to_buffer(sg, 2, zstrm->local_copy, src_len); + req.src = zstrm->local_copy; + } + + ret = comp->ops->decompress(comp->params, &zstrm->ctx, &req); + if (src) + kunmap_local(src); + return ret; } int zcomp_cpu_up_prepare(unsigned int cpu, struct hlist_node *node) diff --git a/drivers/block/zram/zcomp.h b/drivers/block/zram/zcomp.h index 81a0f3f6ff48..f38fd31f9e4e 100644 --- a/drivers/block/zram/zcomp.h +++ b/drivers/block/zram/zcomp.h @@ -5,6 +5,8 @@ #include +struct scatterlist; + #define ZCOMP_PARAM_NOT_SET INT_MIN struct deflate_params { @@ -91,6 +93,6 @@ void zcomp_stream_put(struct zcomp_strm *zstrm); int zcomp_compress(struct zcomp *comp, struct zcomp_strm *zstrm, const void *src, unsigned int *dst_len); int zcomp_decompress(struct zcomp *comp, struct zcomp_strm *zstrm, - const void *src, unsigned int src_len, void *dst); + struct scatterlist *sg, unsigned int src_len, void *dst); #endif /* _ZCOMP_H_ */ diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index 2359eaa6f531..024402438bd1 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include @@ -1347,9 +1348,9 @@ static int decompress_bdev_page(struct zram *zram, struct page *page, unsigned long index) { struct zcomp_strm *zstrm; + struct scatterlist sg[1]; unsigned int size; int ret, prio; - void *src; slot_lock(zram, index); /* Since slot was unlocked we need to make sure it's still ZRAM_WB */ @@ -1368,13 +1369,18 @@ static int decompress_bdev_page(struct zram *zram, struct page *page, size = get_slot_size(zram, index); prio = get_slot_comp_priority(zram, index); + sg_init_table(sg, 1); + sg_set_page(sg, page, size, 0); + zstrm = zcomp_stream_get(zram->comps[prio]); - src = kmap_local_page(page); - ret = zcomp_decompress(zram->comps[prio], zstrm, src, size, + ret = zcomp_decompress(zram->comps[prio], zstrm, sg, size, zstrm->local_copy); - if (!ret) - copy_page(src, zstrm->local_copy); - kunmap_local(src); + if (!ret) { + void *dst = kmap_local_page(page); + + copy_page(dst, zstrm->local_copy); + kunmap_local(dst); + } zcomp_stream_put(zstrm); slot_unlock(zram, index); @@ -2086,15 +2092,19 @@ static int read_same_filled_page(struct zram *zram, struct page *page, static int read_incompressible_page(struct zram *zram, struct page *page, unsigned long index) { + struct scatterlist sg[2]; unsigned long handle; void *src, *dst; handle = get_slot_handle(zram, index); - src = zs_obj_read_begin(zram->mem_pool, handle, PAGE_SIZE, NULL); + zs_obj_read_sg_begin(zram->mem_pool, handle, sg, PAGE_SIZE); + /* an incompressible object never spans two pages */ + src = kmap_local_page(sg_page(sg)); dst = kmap_local_page(page); copy_page(dst, src); kunmap_local(dst); - zs_obj_read_end(zram->mem_pool, handle, PAGE_SIZE, src); + kunmap_local(src); + zs_obj_read_sg_end(zram->mem_pool, handle); return 0; } @@ -2103,9 +2113,10 @@ static int read_compressed_page(struct zram *zram, struct page *page, unsigned long index) { struct zcomp_strm *zstrm; + struct scatterlist sg[2]; unsigned long handle; unsigned int size; - void *src, *dst; + void *dst; int ret, prio; handle = get_slot_handle(zram, index); @@ -2113,12 +2124,11 @@ static int read_compressed_page(struct zram *zram, struct page *page, prio = get_slot_comp_priority(zram, index); zstrm = zcomp_stream_get(zram->comps[prio]); - src = zs_obj_read_begin(zram->mem_pool, handle, size, - zstrm->local_copy); + zs_obj_read_sg_begin(zram->mem_pool, handle, sg, size); dst = kmap_local_page(page); - ret = zcomp_decompress(zram->comps[prio], zstrm, src, size, dst); + ret = zcomp_decompress(zram->comps[prio], zstrm, sg, size, dst); kunmap_local(dst); - zs_obj_read_end(zram->mem_pool, handle, size, src); + zs_obj_read_sg_end(zram->mem_pool, handle); zcomp_stream_put(zstrm); return ret; @@ -2128,25 +2138,23 @@ static int read_compressed_page(struct zram *zram, struct page *page, static int read_from_zspool_raw(struct zram *zram, struct page *page, unsigned long index) { - struct zcomp_strm *zstrm; + struct scatterlist sg[2]; unsigned long handle; unsigned int size; - void *src; + void *dst; handle = get_slot_handle(zram, index); size = get_slot_size(zram, index); /* - * We need to get stream just for ->local_copy buffer, in - * case if object spans two physical pages. No decompression - * takes place here, as we read raw compressed data. + * No decompression takes place here, we copy out raw compressed + * data directly into the destination page. */ - zstrm = zcomp_stream_get(zram->comps[ZRAM_PRIMARY_COMP]); - src = zs_obj_read_begin(zram->mem_pool, handle, size, - zstrm->local_copy); - memcpy_to_page(page, 0, src, size); - zs_obj_read_end(zram->mem_pool, handle, size, src); - zcomp_stream_put(zstrm); + zs_obj_read_sg_begin(zram->mem_pool, handle, sg, size); + dst = kmap_local_page(page); + sg_copy_to_buffer(sg, sg_nents(sg), dst, size); + kunmap_local(dst); + zs_obj_read_sg_end(zram->mem_pool, handle); memzero_page(page, size, PAGE_SIZE - size);