From 3a684e8037c20e18e31cfbff3296afee3cb86ab9 Mon Sep 17 00:00:00 2001 From: Chris Goldsworthy Date: Wed, 20 Apr 2022 16:12:16 -0700 Subject: [PATCH] dma-heap: qcom: Take a snapshot of our DMA-BUF heap driver from msm-5.15 Take a snapshot of our DMA-BUF heap driver as of commit f843a80d2c45 ("Merge "defconfig: gen3auto: Separated if_changed"") on msm-5.15. On top of taking the snapshot, make the following changes: - Remove the references to the deferred free helper, which will be added later. - Remove the references to the pool size getters, which will be added later. - Add the deferred mapping driver. - Remove the heap helper buffer, and associated headers / inclusions (these are unused). - Remove the bitstream contig heap as this is no longer required. Change-Id: Ib98b08139d11940f4f0030294c6c7c18897f6531 Signed-off-by: Chris Goldsworthy --- drivers/dma-buf/heaps/Kconfig | 63 ++ drivers/dma-buf/heaps/Makefile | 11 + drivers/dma-buf/heaps/qcom_carveout_heap.c | 589 ++++++++++++++ drivers/dma-buf/heaps/qcom_carveout_heap.h | 25 + drivers/dma-buf/heaps/qcom_cma_heap.c | 198 +++++ drivers/dma-buf/heaps/qcom_cma_heap.h | 20 + drivers/dma-buf/heaps/qcom_dma_heap.c | 98 +++ drivers/dma-buf/heaps/qcom_dma_heap_priv.h | 45 ++ .../heaps/qcom_dma_heap_secure_utils.c | 272 +++++++ .../heaps/qcom_dma_heap_secure_utils.h | 24 + drivers/dma-buf/heaps/qcom_dt_parser.c | 203 +++++ drivers/dma-buf/heaps/qcom_dt_parser.h | 57 ++ .../dma-buf/heaps/qcom_dynamic_page_pool.c | 332 ++++++++ .../dma-buf/heaps/qcom_dynamic_page_pool.h | 110 +++ .../dma-buf/heaps/qcom_secure_system_heap.c | 716 ++++++++++++++++++ .../dma-buf/heaps/qcom_secure_system_heap.h | 30 + drivers/dma-buf/heaps/qcom_sg_ops.c | 544 +++++++++++++ drivers/dma-buf/heaps/qcom_sg_ops.h | 43 ++ drivers/dma-buf/heaps/qcom_system_heap.c | 584 ++++++++++++++ drivers/dma-buf/heaps/qcom_system_heap.h | 37 + drivers/iommu/msm_dma_iommu_mapping.c | 480 ++++++++++++ include/linux/msm_dma_iommu_mapping.h | 144 ++++ include/linux/qcom_dma_heap.h | 108 +++ include/linux/qcom_dma_heap_dt_constants.h | 12 + include/uapi/linux/msm_ion.h | 95 +++ include/uapi/linux/msm_ion_ids.h | 37 + 26 files changed, 4877 insertions(+) create mode 100644 drivers/dma-buf/heaps/qcom_carveout_heap.c create mode 100644 drivers/dma-buf/heaps/qcom_carveout_heap.h create mode 100644 drivers/dma-buf/heaps/qcom_cma_heap.c create mode 100644 drivers/dma-buf/heaps/qcom_cma_heap.h create mode 100644 drivers/dma-buf/heaps/qcom_dma_heap.c create mode 100644 drivers/dma-buf/heaps/qcom_dma_heap_priv.h create mode 100644 drivers/dma-buf/heaps/qcom_dma_heap_secure_utils.c create mode 100644 drivers/dma-buf/heaps/qcom_dma_heap_secure_utils.h create mode 100644 drivers/dma-buf/heaps/qcom_dt_parser.c create mode 100644 drivers/dma-buf/heaps/qcom_dt_parser.h create mode 100644 drivers/dma-buf/heaps/qcom_dynamic_page_pool.c create mode 100644 drivers/dma-buf/heaps/qcom_dynamic_page_pool.h create mode 100644 drivers/dma-buf/heaps/qcom_secure_system_heap.c create mode 100644 drivers/dma-buf/heaps/qcom_secure_system_heap.h create mode 100644 drivers/dma-buf/heaps/qcom_sg_ops.c create mode 100644 drivers/dma-buf/heaps/qcom_sg_ops.h create mode 100644 drivers/dma-buf/heaps/qcom_system_heap.c create mode 100644 drivers/dma-buf/heaps/qcom_system_heap.h create mode 100644 drivers/iommu/msm_dma_iommu_mapping.c create mode 100644 include/linux/msm_dma_iommu_mapping.h create mode 100644 include/linux/qcom_dma_heap.h create mode 100644 include/linux/qcom_dma_heap_dt_constants.h create mode 100644 include/uapi/linux/msm_ion.h create mode 100644 include/uapi/linux/msm_ion_ids.h diff --git a/drivers/dma-buf/heaps/Kconfig b/drivers/dma-buf/heaps/Kconfig index e273fb18feca..c6f36077dfd3 100644 --- a/drivers/dma-buf/heaps/Kconfig +++ b/drivers/dma-buf/heaps/Kconfig @@ -12,3 +12,66 @@ config DMABUF_HEAPS_CMA Choose this option to enable dma-buf CMA heap. This heap is backed by the Contiguous Memory Allocator (CMA). If your system has these regions, you should say Y here. + +config QCOM_DMABUF_HEAPS + tristate "QCOM DMA-BUF Heaps" + depends on DMABUF_HEAPS + help + Choose this option to build the dma-buf heap module. Other + heaps can subsequently be compiled into this module by enabling + the appropriate defconfig option. + If in doubt, say M here. + +config QCOM_DMABUF_HEAPS_SYSTEM + bool "QCOM DMA-BUF System Heap" + depends on QCOM_DMABUF_HEAPS + help + Choose this option to build the QCOM DMA-BUF system + heap. This heap combines the built-in system heap + along with other optimizatons. If in doubt, say Y + here. + +config QCOM_DMABUF_HEAPS_PAGE_POOL_REFILL + bool "QCOM DMA-BUF Page Pool Refill" + depends on QCOM_DMABUF_HEAPS_SYSTEM + help + Choose this option to enable the page pool auto refill + feature for the system heap's page pools. When enabled, + a thread is created which wakes up to refill the page pools + when they fall below a particular threshold, which in turn + results in smaller allocation latencies. If in doubt, say Y + here. + +config QCOM_DMABUF_HEAPS_SYSTEM_SECURE + bool "QCOM DMA-BUF System Secure Heap" + depends on QCOM_DMABUF_HEAPS && QCOM_SECURE_BUFFER + depends on QCOM_DMABUF_HEAPS_SYSTEM + help + Choose this option to build the QCOM DMA-BUF system + secure heap. This heap, which combines the built-in + system heap along with other optimizations, assigns + memory that is secure by default. If in doubt, say Y + here. + +config QCOM_DMABUF_HEAPS_SYSTEM_UNCACHED + bool "QCOM DMA-BUF Uncached System Heap" + depends on QCOM_DMABUF_HEAPS && QCOM_DMABUF_HEAPS_SYSTEM + help + Choose this option to create an uncached QCOM DMA-BUF + system heap. This heap is equivalent to our system heap, + such that the memory is mapped as uncached. If in doubt, + say Y here. + +config QCOM_DMABUF_HEAPS_CMA + bool "QCOM DMA-BUF CMA Heap" + depends on QCOM_DMABUF_HEAPS && DMA_CMA + help + Choose this option to build the QCOM DMA-BUF CMA heap. + If in doubt, say Y here. + +config QCOM_DMABUF_HEAPS_CARVEOUT + bool "QCOM dma-buf Carveout Heap" + depends on QCOM_DMABUF_HEAPS + help + Choose this option to build the QCOM DMA-BUF Carveout heap. + If in doubt, say Y here. diff --git a/drivers/dma-buf/heaps/Makefile b/drivers/dma-buf/heaps/Makefile index 974467791032..45d27fbd446a 100644 --- a/drivers/dma-buf/heaps/Makefile +++ b/drivers/dma-buf/heaps/Makefile @@ -1,3 +1,14 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DMABUF_HEAPS_SYSTEM) += system_heap.o obj-$(CONFIG_DMABUF_HEAPS_CMA) += cma_heap.o +obj-$(CONFIG_QCOM_DMABUF_HEAPS) += qcom_dma_heaps.o +qcom_dma_heaps-y := qcom_dma_heap.o qcom_dt_parser.o qcom_sg_ops.o +qcom_dma_heaps-$(CONFIG_QCOM_DMABUF_HEAPS_SYSTEM) += qcom_system_heap.o \ + qcom_dynamic_page_pool.o +qcom_dma_heaps-$(CONFIG_QCOM_DMABUF_HEAPS_SYSTEM_SECURE) += qcom_secure_system_heap.o \ + qcom_dynamic_page_pool.o \ + qcom_dma_heap_secure_utils.o +qcom_dma_heaps-$(CONFIG_QCOM_DMABUF_HEAPS_CMA) += qcom_cma_heap.o +qcom_dma_heaps-$(CONFIG_QCOM_DMABUF_HEAPS_CARVEOUT) += qcom_carveout_heap.o \ + qcom_dma_heap_secure_utils.o +qcom_dma_heaps-$(CONFIG_QCOM_DMABUF_HEAPS_BITSTREAM_CONTIG) += qcom_bitstream_contig_heap.o diff --git a/drivers/dma-buf/heaps/qcom_carveout_heap.c b/drivers/dma-buf/heaps/qcom_carveout_heap.c new file mode 100644 index 000000000000..511ae29557b2 --- /dev/null +++ b/drivers/dma-buf/heaps/qcom_carveout_heap.c @@ -0,0 +1,589 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DMA-BUF heap carveout heap allocator. Copied from + * drivers/staging/android/ion/heaps/ion_carveout_heap.c as of commit + * aeb022cc01ecc ("dma-heap: qcom: Change symbol names to let module be built + * in") + * + * Copyright (C) 2011 Google, Inc. + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "qcom_dma_heap_secure_utils.h" +#include "qcom_sg_ops.h" +#include "qcom_carveout_heap.h" + +#define CARVEOUT_ALLOCATE_FAIL -1 + + +/* + * @pool_refcount_priv - + * Cookie set by carveout_heap_add_memory for use with its callbacks. + * Cookie provider will call carveout_heap_remove_memory if refcount + * reaches zero. + * @pool_refcount_get - + * Function callback to Increase refcount. Returns 0 + * on success and fails if refcount is already zero. + * @pool_refcount_put - Function callback to decrease refcount. + */ +struct carveout_heap { + struct dma_heap *heap; + struct rw_semaphore mem_sem; + struct gen_pool *pool; + struct device *dev; + bool is_secure; + bool is_nomap; + phys_addr_t base; + + void *pool_refcount_priv; + int (*pool_refcount_get)(void *priv); + void (*pool_refcount_put)(void *priv); +}; + +struct secure_carveout_heap { + u32 token; + struct carveout_heap carveout_heap; +}; + +static void sc_heap_free(struct qcom_sg_buffer *buffer); + +void pages_sync_for_device(struct device *dev, struct page *page, + size_t size, enum dma_data_direction dir) +{ + struct scatterlist sg; + + sg_init_table(&sg, 1); + sg_set_page(&sg, page, size, 0); + /* + * This is not correct - sg_dma_address needs a dma_addr_t that is valid + * for the targeted device, but this works on the currently targeted + * hardware. + */ + sg_dma_address(&sg) = page_to_phys(page); + dma_sync_sg_for_device(dev, &sg, 1, dir); +} + +static int carveout_pool_refcount_get(struct carveout_heap *carveout_heap) +{ + if (!carveout_heap->pool_refcount_get) + return 0; + + return carveout_heap->pool_refcount_get(carveout_heap->pool_refcount_priv); +} + +static void carveout_pool_refcount_put(struct carveout_heap *carveout_heap) +{ + if (!carveout_heap->pool_refcount_put) + return; + + carveout_heap->pool_refcount_put(carveout_heap->pool_refcount_priv); +} + +static phys_addr_t carveout_allocate(struct carveout_heap *carveout_heap, + unsigned long size) +{ + unsigned long offset = CARVEOUT_ALLOCATE_FAIL; + + down_read(&carveout_heap->mem_sem); + if (carveout_heap->pool) { + if (carveout_pool_refcount_get(carveout_heap)) + goto unlock; + + offset = gen_pool_alloc(carveout_heap->pool, size); + if (!offset) { + offset = CARVEOUT_ALLOCATE_FAIL; + carveout_pool_refcount_put(carveout_heap); + goto unlock; + } + } + +unlock: + up_read(&carveout_heap->mem_sem); + return offset; +} + +static void carveout_free(struct carveout_heap *carveout_heap, + phys_addr_t addr, unsigned long size) +{ + if (addr == CARVEOUT_ALLOCATE_FAIL) + return; + + down_read(&carveout_heap->mem_sem); + if (carveout_heap->pool) + gen_pool_free(carveout_heap->pool, addr, size); + carveout_pool_refcount_put(carveout_heap); + up_read(&carveout_heap->mem_sem); +} + +struct mem_buf_vmperm * +carveout_setup_vmperm(struct carveout_heap *carveout_heap, + struct sg_table *sgt) +{ + struct secure_carveout_heap *sc_heap; + struct mem_buf_vmperm *vmperm; + int *vmids, *perms; + u32 nr; + int ret; + + if (!carveout_heap->is_secure) { + vmperm = mem_buf_vmperm_alloc(sgt); + return vmperm; + } + + sc_heap = container_of(carveout_heap, + struct secure_carveout_heap, carveout_heap); + + ret = get_vmperm_from_ion_flags(sc_heap->token, + &vmids, &perms, &nr); + if (ret) + return ERR_PTR(ret); + + vmperm = mem_buf_vmperm_alloc_staticvm(sgt, vmids, perms, nr); + kfree(vmids); + kfree(perms); + + return vmperm; +} + +static struct dma_buf *__carveout_heap_allocate(struct carveout_heap *carveout_heap, + unsigned long len, + unsigned long fd_flags, + unsigned long heap_flags, + void (*buffer_free)(struct qcom_sg_buffer *)) +{ + struct sg_table *table; + struct qcom_sg_buffer *buffer; + phys_addr_t paddr; + int ret; + DEFINE_DMA_BUF_EXPORT_INFO(exp_info); + struct dma_buf *dmabuf; + struct device *dev = carveout_heap->dev; + + buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); + if (!buffer) + return ERR_PTR(-ENOMEM); + + /* Initialize the buffer */ + INIT_LIST_HEAD(&buffer->attachments); + mutex_init(&buffer->lock); + buffer->heap = carveout_heap->heap; + buffer->len = len; + buffer->free = buffer_free; + if (carveout_heap->is_nomap) + buffer->uncached = true; + + table = &buffer->sg_table; + ret = sg_alloc_table(table, 1, GFP_KERNEL); + if (ret) + goto err_free; + + paddr = carveout_allocate(carveout_heap, len); + if (paddr == CARVEOUT_ALLOCATE_FAIL) { + ret = -ENOMEM; + goto err_free_table; + } + + sg_set_page(table->sgl, pfn_to_page(PFN_DOWN(paddr)), len, 0); + + if (!carveout_heap->is_secure && !carveout_heap->is_nomap) + pages_sync_for_device(dev, sg_page(table->sgl), + buffer->len, DMA_FROM_DEVICE); + + buffer->vmperm = carveout_setup_vmperm(carveout_heap, &buffer->sg_table); + if (IS_ERR(buffer->vmperm)) + goto err_free_carveout; + + + /* Instantiate our dma_buf */ + exp_info.exp_name = dma_heap_get_name(carveout_heap->heap); + exp_info.size = buffer->len; + exp_info.flags = fd_flags; + exp_info.priv = buffer; + dmabuf = mem_buf_dma_buf_export(&exp_info, &qcom_sg_buf_ops); + if (IS_ERR(dmabuf)) { + ret = PTR_ERR(dmabuf); + goto err_free_vmperm; + } + + return dmabuf; + +err_free_vmperm: + mem_buf_vmperm_release(buffer->vmperm); +err_free_carveout: + carveout_free(carveout_heap, paddr, len); +err_free_table: + sg_free_table(table); +err_free: + kfree(buffer); + return ERR_PTR(ret); +} + +static int carveout_pages_zero(struct page *page, size_t size, pgprot_t prot); + +static void carveout_heap_free(struct qcom_sg_buffer *buffer) +{ + struct carveout_heap *carveout_heap; + struct sg_table *table = &buffer->sg_table; + struct page *page = sg_page(table->sgl); + phys_addr_t paddr = page_to_phys(page); + struct device *dev; + + carveout_heap = dma_heap_get_drvdata(buffer->heap); + + dev = carveout_heap->dev; + + if (!carveout_heap->is_nomap) { + carveout_pages_zero(page, buffer->len, PAGE_KERNEL); + pages_sync_for_device(dev, page, + buffer->len, DMA_BIDIRECTIONAL); + } else { + carveout_pages_zero(page, buffer->len, pgprot_writecombine(PAGE_KERNEL)); + } + + carveout_free(carveout_heap, paddr, buffer->len); + sg_free_table(table); + kfree(buffer); +} + + +static struct dma_buf *carveout_heap_allocate(struct dma_heap *heap, + unsigned long len, + unsigned long fd_flags, + unsigned long heap_flags) +{ + struct carveout_heap *carveout_heap = dma_heap_get_drvdata(heap); + + return __carveout_heap_allocate(carveout_heap, len, fd_flags, + heap_flags, carveout_heap_free); +} + +static int carveout_heap_clear_pages(struct page **pages, int num, pgprot_t prot) +{ + void *addr = vmap(pages, num, VM_MAP, prot); + + if (!addr) + return -ENOMEM; + memset(addr, 0, PAGE_SIZE * num); + vunmap(addr); + + return 0; +} + +static int carveout_heap_sglist_zero(struct scatterlist *sgl, unsigned int nents, pgprot_t prot) +{ + int p = 0; + int ret = 0; + struct sg_page_iter piter; + struct page *pages[32]; + + for_each_sg_page(sgl, &piter, nents, 0) { + pages[p++] = sg_page_iter_page(&piter); + if (p == ARRAY_SIZE(pages)) { + ret = carveout_heap_clear_pages(pages, p, prot); + if (ret) + return ret; + p = 0; + } + } + if (p) + ret = carveout_heap_clear_pages(pages, p, prot); + + return ret; +} + +static int carveout_pages_zero(struct page *page, size_t size, pgprot_t prot) +{ + struct scatterlist sg; + + sg_init_table(&sg, 1); + sg_set_page(&sg, page, size, 0); + return carveout_heap_sglist_zero(&sg, 1, prot); +} + +static int carveout_init_heap_memory(struct carveout_heap *co_heap, + phys_addr_t base, ssize_t size, + bool sync) +{ + struct page *page = pfn_to_page(PFN_DOWN(base)); + struct device *dev = co_heap->dev; + int ret = 0; + + if (sync) { + if (!pfn_valid(PFN_DOWN(base))) + return -EINVAL; + pages_sync_for_device(dev, page, size, DMA_BIDIRECTIONAL); + } + + ret = carveout_pages_zero(page, size, pgprot_writecombine(PAGE_KERNEL)); + if (ret) + return ret; + + co_heap->pool = gen_pool_create(PAGE_SHIFT, -1); + if (!co_heap->pool) + return -ENOMEM; + + co_heap->base = base; + gen_pool_add(co_heap->pool, co_heap->base, size, -1); + + return 0; +} + +int carveout_heap_add_memory(char *heap_name, struct sg_table *sgt, void *cookie, + int (*get)(void *), void (*put)(void *)) +{ + struct dma_heap *heap; + struct carveout_heap *carveout_heap; + int ret; + + if (!sgt || sgt->nents != 1) + return -EINVAL; + + heap = dma_heap_find(heap_name); + if (!heap) { + pr_err_ratelimited("%s: No heap named %s\n", __func__, heap_name); + return -EINVAL; + } + + if (!get || !put || !cookie) { + pr_err_ratelimited("%s: Missing refcount callbacks\n", __func__); + return -EINVAL; + } + + carveout_heap = dma_heap_get_drvdata(heap); + + down_write(&carveout_heap->mem_sem); + if (carveout_heap->pool) { + ret = -EBUSY; + goto unlock; + } + + ret = carveout_init_heap_memory(carveout_heap, + page_to_phys(sg_page(sgt->sgl)), + sgt->sgl->length, true); + if (ret) + goto unlock; + + carveout_heap->pool_refcount_priv = cookie; + carveout_heap->pool_refcount_get = get; + carveout_heap->pool_refcount_put = put; + +unlock: + up_write(&carveout_heap->mem_sem); + return ret; +} +EXPORT_SYMBOL(carveout_heap_add_memory); + +int carveout_heap_remove_memory(char *heap_name, + struct sg_table *sgt) +{ + struct dma_heap *heap; + struct carveout_heap *carveout_heap; + phys_addr_t base; + int ret = 0; + + if (!sgt || sgt->nents != 1) + return -EINVAL; + + heap = dma_heap_find(heap_name); + if (!heap) + return -EINVAL; + + carveout_heap = dma_heap_get_drvdata(heap); + + down_write(&carveout_heap->mem_sem); + if (!carveout_heap->pool) { + ret = -EINVAL; + goto unlock; + } + + base = page_to_phys(sg_page(sgt->sgl)); + if (carveout_heap->base != base) { + ret = 0; + goto unlock; + } + + if (gen_pool_size(carveout_heap->pool) != + gen_pool_avail(carveout_heap->pool)) { + ret = -EBUSY; + goto unlock; + } + + gen_pool_destroy(carveout_heap->pool); + carveout_heap->pool = NULL; + carveout_heap->base = 0; + carveout_heap->pool_refcount_priv = NULL; + carveout_heap->pool_refcount_get = NULL; + carveout_heap->pool_refcount_put = NULL; +unlock: + up_write(&carveout_heap->mem_sem); + return ret; +} +EXPORT_SYMBOL(carveout_heap_remove_memory); + +static int __carveout_heap_init(struct platform_heap *heap_data, + struct carveout_heap *carveout_heap, + bool sync) +{ + struct device *dev = heap_data->dev; + bool dynamic_heap = heap_data->is_dynamic; + int ret = 0; + + carveout_heap->dev = dev; + if (!dynamic_heap) + ret = carveout_init_heap_memory(carveout_heap, + heap_data->base, + heap_data->size, sync); + + init_rwsem(&carveout_heap->mem_sem); + + return ret; +} + +static const struct dma_heap_ops carveout_heap_ops = { + .allocate = carveout_heap_allocate, +}; + +static void carveout_heap_destroy(struct carveout_heap *heap); + +int qcom_carveout_heap_create(struct platform_heap *heap_data) +{ + struct dma_heap_export_info exp_info; + struct carveout_heap *carveout_heap; + int ret; + + carveout_heap = kzalloc(sizeof(*carveout_heap), GFP_KERNEL); + if (!carveout_heap) + return -ENOMEM; + + ret = __carveout_heap_init(heap_data, carveout_heap, !heap_data->is_nomap); + if (ret) + goto err; + + carveout_heap->is_secure = false; + carveout_heap->is_nomap = heap_data->is_nomap; + + exp_info.name = heap_data->name; + exp_info.ops = &carveout_heap_ops; + exp_info.priv = carveout_heap; + + carveout_heap->heap = dma_heap_add(&exp_info); + if (IS_ERR(carveout_heap->heap)) { + ret = PTR_ERR(carveout_heap->heap); + goto destroy_heap; + } + + return 0; + +destroy_heap: + carveout_heap_destroy(carveout_heap); +err: + kfree(carveout_heap); + + return ret; +} + +static void carveout_heap_destroy(struct carveout_heap *carveout_heap) +{ + down_write(&carveout_heap->mem_sem); + if (carveout_heap->pool) + gen_pool_destroy(carveout_heap->pool); + up_write(&carveout_heap->mem_sem); + carveout_heap = NULL; +} + +static struct dma_buf *sc_heap_allocate(struct dma_heap *heap, + unsigned long len, + unsigned long fd_flags, + unsigned long heap_flags) +{ + struct secure_carveout_heap *sc_heap; + + sc_heap = dma_heap_get_drvdata(heap); + return __carveout_heap_allocate(&sc_heap->carveout_heap, len, + fd_flags, heap_flags, sc_heap_free); +} + +static void sc_heap_free(struct qcom_sg_buffer *buffer) +{ + struct secure_carveout_heap *sc_heap; + struct sg_table *table = &buffer->sg_table; + struct page *page = sg_page(table->sgl); + phys_addr_t paddr = PFN_PHYS(page_to_pfn(page)); + + sc_heap = dma_heap_get_drvdata(buffer->heap); + + if (qcom_is_buffer_hlos_accessible(sc_heap->token)) { + if (!buffer->uncached) + carveout_pages_zero(page, buffer->len, PAGE_KERNEL); + else + carveout_pages_zero(page, buffer->len, pgprot_writecombine(PAGE_KERNEL)); + } + carveout_free(&sc_heap->carveout_heap, paddr, buffer->len); + sg_free_table(table); + kfree(buffer); +} + +static struct dma_heap_ops sc_heap_ops = { + .allocate = sc_heap_allocate, +}; + +int qcom_secure_carveout_heap_create(struct platform_heap *heap_data) +{ + struct dma_heap_export_info exp_info; + struct secure_carveout_heap *sc_heap; + int ret; + + sc_heap = kzalloc(sizeof(*sc_heap), GFP_KERNEL); + if (!sc_heap) + return -ENOMEM; + + ret = __carveout_heap_init(heap_data, &sc_heap->carveout_heap, false); + if (ret) + goto err; + + ret = hyp_assign_from_flags(heap_data->base, heap_data->size, + heap_data->token); + if (ret) { + pr_err("secure_carveout_heap: Assign token 0x%x failed\n", + heap_data->token); + goto destroy_heap; + } + + sc_heap->token = heap_data->token; + sc_heap->carveout_heap.is_secure = true; + + exp_info.name = heap_data->name; + exp_info.ops = &sc_heap_ops; + exp_info.priv = sc_heap; + + sc_heap->carveout_heap.heap = dma_heap_add(&exp_info); + if (IS_ERR(sc_heap->carveout_heap.heap)) { + ret = PTR_ERR(sc_heap->carveout_heap.heap); + goto destroy_heap; + } + + return 0; + +destroy_heap: + carveout_heap_destroy(&sc_heap->carveout_heap); +err: + kfree(sc_heap); + + return ret; +} diff --git a/drivers/dma-buf/heaps/qcom_carveout_heap.h b/drivers/dma-buf/heaps/qcom_carveout_heap.h new file mode 100644 index 000000000000..6bd8c3478dae --- /dev/null +++ b/drivers/dma-buf/heaps/qcom_carveout_heap.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#ifndef _QCOM_CARVEOUT_HEAP_H +#define _QCOM_CARVEOUT_HEAP_H + +#include "qcom_dt_parser.h" + +#ifdef CONFIG_QCOM_DMABUF_HEAPS_CARVEOUT +int qcom_secure_carveout_heap_create(struct platform_heap *heap_data); +int qcom_carveout_heap_create(struct platform_heap *heap_data); +#else +static int qcom_secure_carveout_heap_create(struct platform_heap *heap_data) +{ + return 1; +} +static int qcom_carveout_heap_create(struct platform_heap *heap_data) +{ + return 1; +} +#endif + +#endif /* _QCOM_CARVEOUT_HEAP_H */ diff --git a/drivers/dma-buf/heaps/qcom_cma_heap.c b/drivers/dma-buf/heaps/qcom_cma_heap.c new file mode 100644 index 000000000000..637bf222b06e --- /dev/null +++ b/drivers/dma-buf/heaps/qcom_cma_heap.c @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DMABUF CMA heap exporter + * Copied from drivers/dma-buf/heaps/cma_heap.c as of commit b61614ec318a + * ("dma-buf: heaps: Add CMA heap to dmabuf heaps") + * + * Copyright (C) 2012, 2019 Linaro Ltd. + * Author: for ST-Ericsson. + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "qcom_cma_heap.h" +#include "qcom_sg_ops.h" + +struct cma_heap { + struct cma *cma; + /* max_align is in units of page_order, similar to CONFIG_CMA_ALIGNMENT */ + u32 max_align; + bool uncached; +}; + +static void cma_heap_free(struct qcom_sg_buffer *buffer) +{ + struct cma_heap *cma_heap; + unsigned long nr_pages = buffer->len >> PAGE_SHIFT; + struct page *cma_pages = sg_page(buffer->sg_table.sgl); + + cma_heap = dma_heap_get_drvdata(buffer->heap); + + /* free page list */ + sg_free_table(&buffer->sg_table); + /* release memory */ + cma_release(cma_heap->cma, cma_pages, nr_pages); + kfree(buffer); +} + +/* dmabuf heap CMA operations functions */ +struct dma_buf *cma_heap_allocate(struct dma_heap *heap, + unsigned long len, + unsigned long fd_flags, + unsigned long heap_flags) +{ + struct cma_heap *cma_heap; + struct qcom_sg_buffer *helper_buffer; + DEFINE_DMA_BUF_EXPORT_INFO(exp_info); + struct page *cma_pages; + size_t size = PAGE_ALIGN(len); + unsigned long nr_pages = size >> PAGE_SHIFT; + unsigned long align = get_order(size); + struct dma_buf *dmabuf; + int ret = -ENOMEM; + + cma_heap = dma_heap_get_drvdata(heap); + + if (align > cma_heap->max_align) + align = cma_heap->max_align; + + helper_buffer = kzalloc(sizeof(*helper_buffer), GFP_KERNEL); + if (!helper_buffer) + return ERR_PTR(-ENOMEM); + + helper_buffer->heap = heap; + INIT_LIST_HEAD(&helper_buffer->attachments); + mutex_init(&helper_buffer->lock); + helper_buffer->len = size; + helper_buffer->uncached = cma_heap->uncached; + helper_buffer->free = cma_heap_free; + + cma_pages = cma_alloc(cma_heap->cma, nr_pages, align, false); + if (!cma_pages) + goto free_buf; + + if (PageHighMem(cma_pages)) { + unsigned long nr_clear_pages = nr_pages; + struct page *page = cma_pages; + + while (nr_clear_pages > 0) { + void *vaddr = kmap_atomic(page); + + memset(vaddr, 0, PAGE_SIZE); + kunmap_atomic(vaddr); + /* + * Avoid wasting time zeroing memory if the process + * has been killed by SIGKILL + */ + if (fatal_signal_pending(current)) + goto free_cma; + + page++; + nr_clear_pages--; + } + } else { + memset(page_address(cma_pages), 0, size); + } + + ret = sg_alloc_table(&helper_buffer->sg_table, 1, GFP_KERNEL); + if (ret) + goto free_cma; + + sg_set_page(helper_buffer->sg_table.sgl, cma_pages, size, 0); + + helper_buffer->vmperm = mem_buf_vmperm_alloc(&helper_buffer->sg_table); + if (IS_ERR(helper_buffer->vmperm)) + goto free_sgtable; + + if (helper_buffer->uncached) { + dma_map_sgtable(dma_heap_get_dev(heap), &helper_buffer->sg_table, + DMA_BIDIRECTIONAL, 0); + dma_unmap_sgtable(dma_heap_get_dev(heap), &helper_buffer->sg_table, + DMA_BIDIRECTIONAL, 0); + } + + /* create the dmabuf */ + exp_info.exp_name = dma_heap_get_name(heap); + exp_info.size = helper_buffer->len; + exp_info.flags = fd_flags; + exp_info.priv = helper_buffer; + dmabuf = mem_buf_dma_buf_export(&exp_info, &qcom_sg_buf_ops); + if (IS_ERR(dmabuf)) { + ret = PTR_ERR(dmabuf); + goto vmperm_release; + } + + return dmabuf; + +vmperm_release: + mem_buf_vmperm_release(helper_buffer->vmperm); +free_sgtable: + sg_free_table(&helper_buffer->sg_table); +free_cma: + cma_release(cma_heap->cma, cma_pages, nr_pages); +free_buf: + kfree(helper_buffer); + return ERR_PTR(ret); +} + +static const struct dma_heap_ops cma_heap_ops = { + .allocate = cma_heap_allocate, +}; + +static int __add_cma_heap(struct platform_heap *heap_data, void *data) +{ + struct cma_heap *cma_heap; + struct dma_heap_export_info exp_info; + struct dma_heap *heap; + + if (!heap_data->dev->cma_area) { + pr_err("%s: CMA area for device uninitialized!\n", __func__); + return -EINVAL; + } + + cma_heap = kzalloc(sizeof(*cma_heap), GFP_KERNEL); + if (!cma_heap) + return -ENOMEM; + + cma_heap->cma = heap_data->dev->cma_area; + cma_heap->max_align = CONFIG_CMA_ALIGNMENT; + if (heap_data->max_align) + cma_heap->max_align = heap_data->max_align; + cma_heap->uncached = heap_data->is_uncached; + + exp_info.name = heap_data->name; + exp_info.ops = &cma_heap_ops; + exp_info.priv = cma_heap; + + heap = dma_heap_add(&exp_info); + if (IS_ERR(heap)) { + int ret = PTR_ERR(heap); + + kfree(cma_heap); + return ret; + } + + if (cma_heap->uncached) + dma_coerce_mask_and_coherent(dma_heap_get_dev(heap), + DMA_BIT_MASK(64)); + + return 0; +} + +int qcom_add_cma_heap(struct platform_heap *heap_data) +{ + return __add_cma_heap(heap_data, NULL); +} diff --git a/drivers/dma-buf/heaps/qcom_cma_heap.h b/drivers/dma-buf/heaps/qcom_cma_heap.h new file mode 100644 index 000000000000..ad40dcff5e8b --- /dev/null +++ b/drivers/dma-buf/heaps/qcom_cma_heap.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#ifndef _QCOM_CMA_HEAP_H +#define _QCOM_CMA_HEAP_H + +#include "qcom_dt_parser.h" + +#ifdef CONFIG_QCOM_DMABUF_HEAPS_CMA +int qcom_add_cma_heap(struct platform_heap *heap_data); +#else +static int qcom_add_cma_heap(struct platform_heap *heap_data) +{ + return 1; +} +#endif + +#endif /* _QCOM_CMA_HEAP_H */ diff --git a/drivers/dma-buf/heaps/qcom_dma_heap.c b/drivers/dma-buf/heaps/qcom_dma_heap.c new file mode 100644 index 000000000000..d5e34a41efcd --- /dev/null +++ b/drivers/dma-buf/heaps/qcom_dma_heap.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include + +#include +#include "qcom_cma_heap.h" +#include "qcom_dt_parser.h" +#include "qcom_system_heap.h" +#include "qcom_carveout_heap.h" +#include "qcom_secure_system_heap.h" + +static int qcom_dma_heap_probe(struct platform_device *pdev) +{ + int ret = 0; + int i; + struct platform_data *heaps; + + qcom_system_heap_create("qcom,system", "system", false); +#ifdef CONFIG_QCOM_DMABUF_HEAPS_SYSTEM_UNCACHED + qcom_system_heap_create("qcom,system-uncached", NULL, true); +#endif + qcom_secure_system_heap_create("qcom,secure-pixel", NULL, + QCOM_DMA_HEAP_FLAG_CP_PIXEL); + qcom_secure_system_heap_create("qcom,secure-non-pixel", NULL, + QCOM_DMA_HEAP_FLAG_CP_NON_PIXEL); + + heaps = parse_heap_dt(pdev); + if (IS_ERR_OR_NULL(heaps)) + return PTR_ERR(heaps); + + for (i = 0; i < heaps->nr; i++) { + struct platform_heap *heap_data = &heaps->heaps[i]; + + switch (heap_data->type) { + case HEAP_TYPE_SECURE_CARVEOUT: + ret = qcom_secure_carveout_heap_create(heap_data); + if (ret < 0) + pr_err("%s: DMA-BUF Heap: Failed to create %s, error is %d\n", + __func__, heap_data->name, ret); + else if (!ret) + pr_info("%s: DMA-BUF Heap: Created %s\n", __func__, + heap_data->name); + break; + case HEAP_TYPE_CARVEOUT: + ret = qcom_carveout_heap_create(heap_data); + if (ret < 0) + pr_err("%s: DMA-BUF Heap: Failed to create %s, error is %d\n", + __func__, heap_data->name, ret); + else if (!ret) + pr_info("%s: DMA-BUF Heap: Created %s\n", __func__, + heap_data->name); + break; + case HEAP_TYPE_CMA: + ret = qcom_add_cma_heap(heap_data); + if (ret < 0) + pr_err("%s: DMA-BUF Heap: Failed to create %s, error is %d\n", + __func__, heap_data->name, ret); + else if (!ret) + pr_info("%s: DMA-BUF Heap: Created %s\n", __func__, + heap_data->name); + break; + default: + pr_err("%s: Unknown heap type %u\n", __func__, heap_data->type); + break; + } + } + + free_pdata(heaps); + + return ret; +} + +static const struct of_device_id qcom_dma_heap_match_table[] = { + {.compatible = "qcom,dma-heaps"}, + {}, +}; + +static struct platform_driver qcom_dma_heap_driver = { + .probe = qcom_dma_heap_probe, + .driver = { + .name = "qcom-dma-heap", + .of_match_table = qcom_dma_heap_match_table, + }, +}; + +static int __init init_heap_driver(void) +{ + return platform_driver_register(&qcom_dma_heap_driver); +} +module_init(init_heap_driver); + +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma-buf/heaps/qcom_dma_heap_priv.h b/drivers/dma-buf/heaps/qcom_dma_heap_priv.h new file mode 100644 index 000000000000..803693e15283 --- /dev/null +++ b/drivers/dma-buf/heaps/qcom_dma_heap_priv.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2011 Google, Inc. + * Copyright (C) 2019 Linaro Ltd. + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#ifndef _QCOM_DMA_HEAP_PRIV_H +#define _QCOM_DMA_HEAP_PRIV_H + +#include +#include + +/** + * struct heap_helper_buffer - helper buffer metadata + * @heap: back pointer to the heap the buffer came from + * @dmabuf: backing dma-buf for this buffer + * @size: size of the buffer + * @priv_virt pointer to heap specific private value + * @lock mutext to protect the data in this structure + * @vmap_cnt count of vmap references on the buffer + * @vaddr vmap'ed virtual address + * @pagecount number of pages in the buffer + * @pages list of page pointers + * @attachments list of device attachments + * + * @free heap callback to free the buffer + */ +struct heap_helper_buffer { + struct dma_heap *heap; + struct dma_buf *dmabuf; + size_t size; + + void *priv_virt; + struct mutex lock; + int vmap_cnt; + void *vaddr; + pgoff_t pagecount; + struct page **pages; + struct list_head attachments; + + void (*free)(struct heap_helper_buffer *buffer); +}; + +#endif /* _QCOM_DMA_HEAP_PRIV_H */ diff --git a/drivers/dma-buf/heaps/qcom_dma_heap_secure_utils.c b/drivers/dma-buf/heaps/qcom_dma_heap_secure_utils.c new file mode 100644 index 000000000000..a6b93819000e --- /dev/null +++ b/drivers/dma-buf/heaps/qcom_dma_heap_secure_utils.c @@ -0,0 +1,272 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include + +int get_secure_vmid(unsigned long flags) +{ + if (flags & QCOM_DMA_HEAP_FLAG_CP_TZ) + return VMID_TZ; + if (flags & QCOM_DMA_HEAP_FLAG_CP_TOUCH) + return VMID_CP_TOUCH; + if (flags & QCOM_DMA_HEAP_FLAG_CP_BITSTREAM) + return VMID_CP_BITSTREAM; + if (flags & QCOM_DMA_HEAP_FLAG_CP_PIXEL) + return VMID_CP_PIXEL; + if (flags & QCOM_DMA_HEAP_FLAG_CP_NON_PIXEL) + return VMID_CP_NON_PIXEL; + if (flags & QCOM_DMA_HEAP_FLAG_CP_CAMERA) + return VMID_CP_CAMERA; + if (flags & QCOM_DMA_HEAP_FLAG_CP_SEC_DISPLAY) + return VMID_CP_SEC_DISPLAY; + if (flags & QCOM_DMA_HEAP_FLAG_CP_APP) + return VMID_CP_APP; + if (flags & QCOM_DMA_HEAP_FLAG_CP_CAMERA_PREVIEW) + return VMID_CP_CAMERA_PREVIEW; + if (flags & QCOM_DMA_HEAP_FLAG_CP_SPSS_SP) + return VMID_CP_SPSS_SP; + if (flags & QCOM_DMA_HEAP_FLAG_CP_SPSS_SP_SHARED) + return VMID_CP_SPSS_SP_SHARED; + if (flags & QCOM_DMA_HEAP_FLAG_CP_SPSS_HLOS_SHARED) + return VMID_CP_SPSS_HLOS_SHARED; + if (flags & QCOM_DMA_HEAP_FLAG_CP_CDSP) + return VMID_CP_CDSP; + if (flags & QCOM_DMA_HEAP_FLAG_CP_MSS_MSA) + return VMID_MSS_MSA; + + return -EINVAL; +} + +static unsigned int count_set_bits(unsigned long val) +{ + return ((unsigned int)bitmap_weight(&val, BITS_PER_LONG)); +} + +bool qcom_is_buffer_hlos_accessible(unsigned long flags) +{ + if (!(flags & QCOM_DMA_HEAP_FLAG_CP_HLOS) && + !(flags & QCOM_DMA_HEAP_FLAG_CP_SPSS_HLOS_SHARED)) + return false; + + return true; +} + +static int get_vmid(unsigned long flags) +{ + int vmid; + + vmid = get_secure_vmid(flags); + if (vmid < 0) { + if (flags & QCOM_DMA_HEAP_FLAG_CP_HLOS) + vmid = VMID_HLOS; + } + return vmid; +} + +static int populate_vm_list(unsigned long flags, unsigned int *vm_list, + int nelems) +{ + unsigned int itr = 0; + int vmid; + + flags = flags & QCOM_DMA_HEAP_FLAGS_CP_MASK; + if (!flags) + return -EINVAL; + + for_each_set_bit(itr, &flags, BITS_PER_LONG) { + vmid = get_vmid(0x1UL << itr); + if (vmid < 0 || !nelems) + return -EINVAL; + + vm_list[nelems - 1] = vmid; + nelems--; + } + return 0; +} + +static int hyp_unassign_sg(struct sg_table *sgt, int *source_vm_list, + int source_nelems, bool clear_page_private) +{ + u32 dest_vmid = VMID_HLOS; + u32 dest_perms = PERM_READ | PERM_WRITE | PERM_EXEC; + struct scatterlist *sg; + int ret, i; + + if (source_nelems <= 0) { + pr_err("%s: source_nelems invalid\n", + __func__); + ret = -EINVAL; + goto out; + } + + ret = hyp_assign_table(sgt, source_vm_list, source_nelems, &dest_vmid, + &dest_perms, 1); + if (ret) + goto out; + + if (clear_page_private) + for_each_sg(sgt->sgl, sg, sgt->nents, i) + ClearPagePrivate(sg_page(sg)); +out: + return ret; +} + +static int hyp_assign_sg(struct sg_table *sgt, int *dest_vm_list, + int dest_nelems, bool set_page_private) +{ + u32 source_vmid = VMID_HLOS; + struct scatterlist *sg; + int *dest_perms; + int ret, i; + + if (dest_nelems <= 0) { + pr_err("%s: dest_nelems invalid\n", + __func__); + ret = -EINVAL; + goto out; + } + + dest_perms = kcalloc(dest_nelems, sizeof(*dest_perms), GFP_KERNEL); + if (!dest_perms) { + ret = -ENOMEM; + goto out; + } + + for (i = 0; i < dest_nelems; i++) + dest_perms[i] = msm_secure_get_vmid_perms(dest_vm_list[i]); + + ret = hyp_assign_table(sgt, &source_vmid, 1, + dest_vm_list, dest_perms, dest_nelems); + + if (ret) { + pr_err("%s: Assign call failed\n", + __func__); + goto out_free_dest; + } + if (set_page_private) + for_each_sg(sgt->sgl, sg, sgt->nents, i) + SetPagePrivate(sg_page(sg)); + +out_free_dest: + kfree(dest_perms); +out: + return ret; +} + +int hyp_unassign_sg_from_flags(struct sg_table *sgt, unsigned long flags, + bool set_page_private) +{ + int ret; + int *source_vm_list; + int source_nelems; + + source_nelems = count_set_bits(flags & QCOM_DMA_HEAP_FLAGS_CP_MASK); + source_vm_list = kcalloc(source_nelems, sizeof(*source_vm_list), + GFP_KERNEL); + if (!source_vm_list) + return -ENOMEM; + ret = populate_vm_list(flags, source_vm_list, source_nelems); + if (ret) { + pr_err("%s: Failed to get secure vmids\n", __func__); + goto out_free_source; + } + + ret = hyp_unassign_sg(sgt, source_vm_list, source_nelems, + set_page_private); + +out_free_source: + kfree(source_vm_list); + return ret; +} + +int hyp_assign_sg_from_flags(struct sg_table *sgt, unsigned long flags, + bool set_page_private) +{ + int ret; + int *dest_vm_list = NULL; + int dest_nelems; + + dest_nelems = count_set_bits(flags & QCOM_DMA_HEAP_FLAGS_CP_MASK); + dest_vm_list = kcalloc(dest_nelems, sizeof(*dest_vm_list), GFP_KERNEL); + if (!dest_vm_list) { + ret = -ENOMEM; + goto out; + } + + ret = populate_vm_list(flags, dest_vm_list, dest_nelems); + if (ret) { + pr_err("%s: Failed to get secure vmid(s)\n", __func__); + goto out_free_dest_vm; + } + + ret = hyp_assign_sg(sgt, dest_vm_list, dest_nelems, + set_page_private); + +out_free_dest_vm: + kfree(dest_vm_list); +out: + return ret; +} + +int get_vmperm_from_ion_flags(unsigned long flags, int **_vmids, + int **_perms, u32 *_nr) +{ + int *vmids, *modes; + u32 nr, i; + + if (!_vmids || !_perms || !_nr) + return -EINVAL; + + nr = count_set_bits(flags); + vmids = kcalloc(nr, sizeof(*vmids), GFP_KERNEL); + if (!vmids) + return -ENOMEM; + + modes = kcalloc(nr, sizeof(*modes), GFP_KERNEL); + if (!modes) { + kfree(vmids); + return -ENOMEM; + } + + if ((flags & ~QCOM_DMA_HEAP_FLAGS_CP_MASK) || + populate_vm_list(flags, vmids, nr)) { + pr_err("%s: Failed to parse secure flags 0x%lx\n", __func__, + flags); + kfree(modes); + kfree(vmids); + return -EINVAL; + } + + for (i = 0; i < nr; i++) + modes[i] = msm_secure_get_vmid_perms(vmids[i]); + + *_vmids = vmids; + *_perms = modes; + *_nr = nr; + return 0; +} + +int hyp_assign_from_flags(u64 base, u64 size, unsigned long flags) +{ + int *vmids, *modes; + u32 nr; + u32 src_vm = VMID_HLOS; + int ret; + + ret = get_vmperm_from_ion_flags(flags, &vmids, &modes, &nr); + if (ret) + return ret; + + ret = hyp_assign_phys(base, size, &src_vm, 1, vmids, modes, nr); + if (ret) + pr_err("%s: Assign call failed, flags 0x%lx\n", __func__, + flags); + + kfree(modes); + kfree(vmids); + return ret; +} diff --git a/drivers/dma-buf/heaps/qcom_dma_heap_secure_utils.h b/drivers/dma-buf/heaps/qcom_dma_heap_secure_utils.h new file mode 100644 index 000000000000..492232e6f796 --- /dev/null +++ b/drivers/dma-buf/heaps/qcom_dma_heap_secure_utils.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#ifndef _QCOM_DMA_HEAP_SECURE_UTILS_H +#define _QCOM_DMA_HEAP_SECURE_UTILS_H + +int hyp_assign_sg_from_flags(struct sg_table *sgt, unsigned long flags, + bool set_page_private); + +int hyp_unassign_sg_from_flags(struct sg_table *sgt, unsigned long flags, + bool set_page_private); + +/* + * This function handles conversion from the legacy ion flag based interface + * to a precise list of vmids and permissions. + */ +int get_vmperm_from_ion_flags(unsigned long flags, int **vmids, int **perms, u32 *nr); +int hyp_assign_from_flags(u64 base, u64 size, unsigned long flags); +bool qcom_is_buffer_hlos_accessible(unsigned long flags); +int get_secure_vmid(unsigned long flags); + +#endif /* _QCOM_DMA_HEAP_SECURE_UTILS_H */ diff --git a/drivers/dma-buf/heaps/qcom_dt_parser.c b/drivers/dma-buf/heaps/qcom_dt_parser.c new file mode 100644 index 000000000000..26e8bb84a113 --- /dev/null +++ b/drivers/dma-buf/heaps/qcom_dt_parser.c @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include "qcom_dt_parser.h" + +#ifdef CONFIG_PANIC_ON_QCOM_DMA_HEAPS_FAILURE +#define QCOM_DMA_HEAP_WARN(fmt...) panic(fmt) +#else /* CONFIG_PANIC_ON_QCOM_DMA_HEAPS_FAILURE */ +#define QCOM_DMA_HEAP_WARN(fmt...) WARN(1, fmt) +#endif /* CONFIG_PANIC_ON_QCOM_DMA_HEAPS_FAILURE */ + +static int populate_heap(struct device_node *node, + struct platform_heap *heap) +{ + int ret; + + /* Mandatory properties */ + ret = of_property_read_string(node, "qcom,dma-heap-name", &heap->name); + if (ret) + goto err; + + ret = of_property_read_u32(node, "qcom,dma-heap-type", &heap->type); + if (ret) { + pr_err("Reading %s property in node %s failed with err %d.\n", + "qcom,dma-heap-type", of_node_full_name(node), ret); + goto err; + } + + /* Optional properties */ + heap->is_dynamic = of_property_read_bool(node, "qcom,dynamic-heap"); + + heap->is_uncached = of_property_read_bool(node, "qcom,uncached-heap"); + + ret = of_property_read_u32(node, "qcom,token", &heap->token); + if (ret && ret != -EINVAL) + goto err; + + ret = of_property_read_u32(node, "qcom,max-align", &heap->max_align); + if (ret && ret != -EINVAL) + goto err; + + return 0; + +err: + if (ret) + QCOM_DMA_HEAP_WARN("%s: Unable to populate heap %s, err: %d\n", + __func__, of_node_full_name(node), ret); + return ret; +} + +void free_pdata(const struct platform_data *pdata) +{ + kfree(pdata->heaps); + kfree(pdata); +} + +static int heap_dt_init(struct device_node *mem_node, + struct platform_heap *heap) +{ + const __be32 *basep; + u64 base, size; + struct device *dev = heap->dev; + struct reserved_mem *rmem; + int ret = 0; + + + rmem = of_reserved_mem_lookup(mem_node); + + if (!rmem) { + dev_err(dev, "Failed to find reserved memory region\n"); + return -EINVAL; + } + + /* + * We only need to call this when the memory-region is managed by + * a reserved memory region driver (e.g. CMA, coherent, etc). In that + * case, they will have ops for device specific initialization for + * the memory region. Otherwise, we have a pure carveout, which needs + * not be initialized. + */ + if (rmem->ops) { + ret = of_reserved_mem_device_init_by_idx(dev, dev->of_node, 0); + if (ret) { + dev_err(dev, + "Failed to initialize memory region rc: %d\n", + ret); + return ret; + } + } + + basep = of_get_address(mem_node, 0, &size, NULL); + if (basep) { + base = of_translate_address(mem_node, basep); + if (base != OF_BAD_ADDR) { + heap->base = base; + heap->size = size; + } else { + ret = -EINVAL; + dev_err(heap->dev, + "Failed to get heap base/size\n"); + of_reserved_mem_device_release(dev); + } + } + heap->is_nomap = of_property_read_bool(mem_node, "no-map"); + + return ret; +} + +static void release_reserved_memory_regions(struct platform_heap *heaps, + int idx) +{ + struct device *dev; + struct device_node *node, *mem_node; + + for (idx = idx - 1; idx >= 0; idx--) { + dev = heaps[idx].dev; + node = dev->of_node; + mem_node = of_parse_phandle(node, "memory-region", 0); + + if (mem_node) + of_reserved_mem_device_release(dev); + + of_node_put(mem_node); + } +} + +struct platform_data *parse_heap_dt(struct platform_device *pdev) +{ + struct platform_data *pdata = NULL; + struct device_node *node; + struct device_node *mem_node; + struct platform_device *new_dev = NULL; + const struct device_node *dt_node = pdev->dev.of_node; + int ret; + int idx = 0; + + pdata = kzalloc(sizeof(*pdata), GFP_KERNEL); + if (!pdata) + return ERR_PTR(-ENOMEM); + + for_each_available_child_of_node(dt_node, node) + pdata->nr++; + + /* + * No heaps defined in the devicetree. However, there may be other + * heaps (e.g. system heaps) that do not need to be defined in the + * devicetree. + */ + if (!pdata->nr) + goto out; + + pdata->heaps = kcalloc(pdata->nr, sizeof(*pdata->heaps), GFP_KERNEL); + if (!pdata->heaps) { + kfree(pdata); + return ERR_PTR(-ENOMEM); + } + + for_each_available_child_of_node(dt_node, node) { + new_dev = of_platform_device_create(node, NULL, &pdev->dev); + if (!new_dev) { + pr_err("Failed to create device %s\n", node->name); + ret = -EINVAL; + goto free_heaps; + } + of_dma_configure(&new_dev->dev, node, true); + + pdata->heaps[idx].dev = &new_dev->dev; + + ret = populate_heap(node, &pdata->heaps[idx]); + if (ret) + goto free_heaps; + + mem_node = of_parse_phandle(node, "memory-region", 0); + if (mem_node) { + ret = heap_dt_init(mem_node, &pdata->heaps[idx]); + if (ret) + goto free_heaps; + + of_node_put(mem_node); + } + + ++idx; + } + +out: + return pdata; + +free_heaps: + of_node_put(node); + release_reserved_memory_regions(pdata->heaps, idx); + free_pdata(pdata); + return ERR_PTR(ret); +} diff --git a/drivers/dma-buf/heaps/qcom_dt_parser.h b/drivers/dma-buf/heaps/qcom_dt_parser.h new file mode 100644 index 000000000000..f1cf66a63f4b --- /dev/null +++ b/drivers/dma-buf/heaps/qcom_dt_parser.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#ifndef _QCOM_DT_PARSER_H +#define _QCOM_DT_PARSER_H + +#include +#include + +/** + * struct platform_heap - defines a heap in the given platform + * @type: type of the heap + * @name: used for debug purposes + * @base: base address of heap in physical memory if applicable + * @size: size of the heap in bytes if applicable + * @dev: the device associated with the heap's DT node + * @is_dynamic: indicates if memory can be added or removed from carveout heaps + * is_nomap: indicates if this heap is assosicated with a memory region which is + * which has no-map property. + * is_uncached: indicates if this heap provides uncached memory or not + * @token: the end points to which memory for secure carveout memory is + * assigned to + * @max_align: page order of the maximum alignment. Used by cma heap. + * + * Provided by the board file. + */ +struct platform_heap { + u32 type; + const char *name; + phys_addr_t base; + size_t size; + struct device *dev; + bool is_uncached; + bool is_dynamic; + bool is_nomap; + u32 token; + u32 max_align; +}; + +/** + * struct platform_data - array of platform heaps passed from board file + * @nr: number of structures in the array + * @heaps: array of platform_heap structions + * + * Provided by the board file in the form of platform data to a platform device. + */ +struct platform_data { + int nr; + struct platform_heap *heaps; +}; + +struct platform_data *parse_heap_dt(struct platform_device *pdev); +void free_pdata(const struct platform_data *pdata); + +#endif /* _QCOM_DT_PARSER_H */ diff --git a/drivers/dma-buf/heaps/qcom_dynamic_page_pool.c b/drivers/dma-buf/heaps/qcom_dynamic_page_pool.c new file mode 100644 index 000000000000..63db70cd71d8 --- /dev/null +++ b/drivers/dma-buf/heaps/qcom_dynamic_page_pool.c @@ -0,0 +1,332 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Dynamic page pool system + * + * Taken from: + * mm/dynamic_page_pool.c + * Git-repo: https://git.linaro.org/people/john.stultz/android-dev.git + * Branch: dma-buf-heap-perf + * Git-commit: 458ea8030852755867bdc0384aa40f97aba7a572 + * + * Based on the ION page pool code + * Copyright (C) 2011 Google, Inc. + * + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include + +#include "qcom_dynamic_page_pool.h" + +static LIST_HEAD(pool_list); +static DEFINE_MUTEX(pool_list_lock); + +void dynamic_page_pool_add(struct dynamic_page_pool *pool, struct page *page) +{ + mutex_lock(&pool->mutex); + if (PageHighMem(page)) { + list_add_tail(&page->lru, &pool->high_items); + pool->high_count++; + } else { + list_add_tail(&page->lru, &pool->low_items); + pool->low_count++; + } + + atomic_inc(&pool->count); + mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE, + 1 << pool->order); + mutex_unlock(&pool->mutex); +} + +struct page *dynamic_page_pool_remove(struct dynamic_page_pool *pool, bool high) +{ + struct page *page; + + if (high) { + BUG_ON(!pool->high_count); + page = list_first_entry(&pool->high_items, struct page, lru); + pool->high_count--; + } else { + BUG_ON(!pool->low_count); + page = list_first_entry(&pool->low_items, struct page, lru); + pool->low_count--; + } + + atomic_dec(&pool->count); + list_del(&page->lru); + mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE, + -(1 << pool->order)); + return page; +} + +void dynamic_page_pool_free(struct dynamic_page_pool *pool, struct page *page) +{ + BUG_ON(pool->order != compound_order(page)); + + dynamic_page_pool_add(pool, page); +} + +int dynamic_page_pool_total(struct dynamic_page_pool *pool, bool high) +{ + int count = pool->low_count; + + if (high) + count += pool->high_count; + + return count << pool->order; +} + +struct dynamic_page_pool *dynamic_page_pool_create(gfp_t gfp_mask, unsigned int order) +{ + struct dynamic_page_pool *pool = kmalloc(sizeof(*pool), GFP_KERNEL); + + if (!pool) + return NULL; + pool->high_count = 0; + pool->low_count = 0; + INIT_LIST_HEAD(&pool->low_items); + INIT_LIST_HEAD(&pool->high_items); + pool->gfp_mask = gfp_mask | __GFP_COMP; + pool->order = order; + mutex_init(&pool->mutex); + + mutex_lock(&pool_list_lock); + list_add(&pool->list, &pool_list); + mutex_unlock(&pool_list_lock); + + return pool; +} + +void dynamic_page_pool_destroy(struct dynamic_page_pool *pool) +{ + struct page *page, *tmp; + LIST_HEAD(pages); + int num_pages = 0; + int ret = DYNAMIC_POOL_SUCCESS; + + /* Remove us from the pool list */ + mutex_lock(&pool_list_lock); + list_del(&pool->list); + mutex_unlock(&pool_list_lock); + + /* Free any remaining pages in the pool */ + mutex_lock(&pool->mutex); + while (true) { + if (pool->low_count) + page = dynamic_page_pool_remove(pool, false); + else if (pool->high_count) + page = dynamic_page_pool_remove(pool, true); + else + break; + + list_add(&page->lru, &pages); + num_pages++; + } + mutex_unlock(&pool->mutex); + + if (num_pages && pool->prerelease_callback) + ret = pool->prerelease_callback(pool, &pages, num_pages); + + if (ret != DYNAMIC_POOL_SUCCESS) { + pr_err("Failed to reclaim pages when destroying the pool!\n"); + return; + } + + list_for_each_entry_safe(page, tmp, &pages, lru) { + list_del(&page->lru); + __free_pages(page, pool->order); + } + + kfree(pool); +} + +static int dynamic_page_pool_do_shrink(struct dynamic_page_pool *pool, gfp_t gfp_mask, + int nr_to_scan) +{ + int freed = 0; + bool high; + struct page *page, *tmp; + LIST_HEAD(pages); + int ret = DYNAMIC_POOL_SUCCESS; + + if (current_is_kswapd()) + high = true; + else + high = !!(gfp_mask & __GFP_HIGHMEM); + + if (nr_to_scan == 0) + return dynamic_page_pool_total(pool, high); + + while (freed < nr_to_scan) { + mutex_lock(&pool->mutex); + if (pool->low_count) { + page = dynamic_page_pool_remove(pool, false); + } else if (high && pool->high_count) { + page = dynamic_page_pool_remove(pool, true); + } else { + mutex_unlock(&pool->mutex); + break; + } + mutex_unlock(&pool->mutex); + list_add(&page->lru, &pages); + freed += (1 << pool->order); + } + + if (freed && pool->prerelease_callback) + ret = pool->prerelease_callback(pool, &pages, freed >> pool->order); + + if (ret != DYNAMIC_POOL_SUCCESS) { + pr_err("Failed to reclaim secure page pool pages!\n"); + return 0; + } + + list_for_each_entry_safe(page, tmp, &pages, lru) { + list_del(&page->lru); + __free_pages(page, pool->order); + } + + return freed; +} + +void dynamic_page_pool_shrink_high_and_low(struct dynamic_page_pool **pools_list, + int num_pools, int nr_to_scan) +{ + int i, remaining = nr_to_scan; + + mutex_lock(&pool_list_lock); + for (i = 0; i < num_pools; i++) { + remaining -= dynamic_page_pool_do_shrink(pools_list[i], __GFP_HIGHMEM, + remaining); + + if (remaining <= 0) { + mutex_unlock(&pool_list_lock); + return; + } + } + mutex_unlock(&pool_list_lock); +} + +static int dynamic_page_pool_shrink(gfp_t gfp_mask, int nr_to_scan) +{ + struct dynamic_page_pool *pool; + int nr_total = 0; + int nr_freed; + int only_scan = 0; + + if (!nr_to_scan) + only_scan = 1; + + mutex_lock(&pool_list_lock); + list_for_each_entry(pool, &pool_list, list) { + if (only_scan) { + nr_total += dynamic_page_pool_do_shrink(pool, + gfp_mask, + nr_to_scan); + } else { + nr_freed = dynamic_page_pool_do_shrink(pool, + gfp_mask, + nr_to_scan); + nr_to_scan -= nr_freed; + nr_total += nr_freed; + if (nr_to_scan <= 0) + break; + } + } + mutex_unlock(&pool_list_lock); + + return nr_total; +} + +static unsigned long dynamic_page_pool_shrink_count(struct shrinker *shrinker, + struct shrink_control *sc) +{ + return dynamic_page_pool_shrink(sc->gfp_mask, 0); +} + +static unsigned long dynamic_page_pool_shrink_scan(struct shrinker *shrinker, + struct shrink_control *sc) +{ + int to_scan = sc->nr_to_scan; + + if (to_scan == 0) + return 0; + + return dynamic_page_pool_shrink(sc->gfp_mask, to_scan); +} + +struct dynamic_page_pool **dynamic_page_pool_create_pools(int vmid, + prerelease_callback callback) +{ + struct dynamic_page_pool **pool_list; + int i; + int ret; + + pool_list = kmalloc_array(NUM_ORDERS, sizeof(*pool_list), GFP_KERNEL); + if (!pool_list) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < NUM_ORDERS; i++) { + pool_list[i] = dynamic_page_pool_create(order_flags[i], + orders[i]); + pool_list[i]->vmid = vmid; + pool_list[i]->prerelease_callback = callback; + atomic_set(&pool_list[i]->count, 0); + pool_list[i]->last_low_watermark_ktime = 0; + + if (IS_ERR_OR_NULL(pool_list[i])) { + int j; + + pr_err("%s: page pool creation failed for the order %u pool!\n", + __func__, orders[i]); + for (j = 0; j < i; j++) + dynamic_page_pool_destroy(pool_list[j]); + + ret = -ENOMEM; + goto free_pool_arr; + } + } + + return pool_list; + +free_pool_arr: + kfree(pool_list); + + return ERR_PTR(ret); +} + +void dynamic_page_pool_release_pools(struct dynamic_page_pool **pool_list) +{ + int i; + + for (i = 0; i < NUM_ORDERS; i++) + dynamic_page_pool_destroy(pool_list[i]); + + kfree(pool_list); +} + +static struct shrinker pool_shrinker = { + .count_objects = dynamic_page_pool_shrink_count, + .scan_objects = dynamic_page_pool_shrink_scan, + .seeks = DEFAULT_SEEKS, + .batch = 0, +}; + +int dynamic_page_pool_init_shrinker(void) +{ + int ret; + static bool registered; + + if (registered) + return 0; + + ret = register_shrinker(&pool_shrinker); + if (ret) + return ret; + + registered = true; + return 0; +} diff --git a/drivers/dma-buf/heaps/qcom_dynamic_page_pool.h b/drivers/dma-buf/heaps/qcom_dynamic_page_pool.h new file mode 100644 index 000000000000..75787b28c1ea --- /dev/null +++ b/drivers/dma-buf/heaps/qcom_dynamic_page_pool.h @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * ION Memory Allocator kernel interface header + * + * Taken from: + * include/linux/dynamic_page_pool.h + * Git-repo: https://git.linaro.org/people/john.stultz/android-dev.git + * Branch: dma-buf-heap-perf + * Git-commit: 458ea8030852755867bdc0384aa40f97aba7a572 + * + * Copyright (C) 2011 Google, Inc. + * + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#ifndef _DYN_PAGE_POOL_H +#define _DYN_PAGE_POOL_H + +#include +#include +#include +#include +#include +#include + +#define HIGH_ORDER_GFP (((GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN \ + | __GFP_NORETRY) & ~__GFP_RECLAIM) \ + | __GFP_COMP) +#define LOW_ORDER_GFP (GFP_HIGHUSER | __GFP_ZERO | __GFP_COMP) + +/* + * We add __GFP_NOWARN for order 4 allocations since the core mm/ framework + * makes no guarantee of these allocations succeeding. + */ +static gfp_t order_flags[] = {HIGH_ORDER_GFP, HIGH_ORDER_GFP, + LOW_ORDER_GFP}; +#if defined(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) && !defined(CONFIG_64BIT) && !defined(CONFIG_ARM_LPAE) +static const unsigned int orders[] = {8, 4, 0}; +#else +static const unsigned int orders[] = {9, 4, 0}; +#endif +#define NUM_ORDERS ARRAY_SIZE(orders) + +enum dynamic_pool_callback_ret { + DYNAMIC_POOL_SUCCESS, + DYNAMIC_POOL_FAILURE, +}; + +struct dynamic_page_pool; + +typedef enum dynamic_pool_callback_ret (*prerelease_callback)(struct dynamic_page_pool *pool, + struct list_head *pages, + int num_pages); + +/** + * struct dynamic_page_pool - pagepool struct + * @high_count: number of highmem items in the pool + * @low_count: number of lowmem items in the pool + * @count: total number of pages/items in the pool + * @high_items: list of highmem items + * @low_items: list of lowmem items + * @last_low_watermark_ktime: most recent time at which the zone watermarks were + * low + * @refill_worker: kthread used to refill a pool, if applicable + * @mutex: lock protecting this struct and especially the count + * item list + * @gfp_mask: gfp_mask to use from alloc + * @order: order of pages in the pool + * @list: list node for list of pools + * @vmid: the vmid used for this pool + * @prerelease_callback: preprocessing function called before pages are + * released to buddy + * + * Allows you to keep a pool of pre allocated pages to use + * Keeping a pool of pages that is ready for dma, ie any cached mapping have + * been invalidated from the cache, provides a significant performance benefit + * on many systems + */ +struct dynamic_page_pool { + int high_count; + int low_count; + atomic_t count; + struct list_head high_items; + struct list_head low_items; + ktime_t last_low_watermark_ktime; + struct task_struct *refill_worker; + struct mutex mutex; + gfp_t gfp_mask; + unsigned int order; + struct list_head list; + int vmid; + prerelease_callback prerelease_callback; +}; + +struct dynamic_page_pool **dynamic_page_pool_create_pools(int vmid, + prerelease_callback callback); +void dynamic_page_pool_release_pools(struct dynamic_page_pool **pool_list); + +struct page *dynamic_page_pool_alloc(struct dynamic_page_pool *pool); +void dynamic_page_pool_free(struct dynamic_page_pool *pool, struct page *page); + +int dynamic_page_pool_init_shrinker(void); +void dynamic_page_pool_shrink_high_and_low(struct dynamic_page_pool **pools_list, + int num_pools, int nr_to_scan); +int dynamic_page_pool_total(struct dynamic_page_pool *pool, bool high); + +struct page *dynamic_page_pool_remove(struct dynamic_page_pool *pool, bool high); +void dynamic_page_pool_add(struct dynamic_page_pool *pool, struct page *page); + +#endif /* _DYN_PAGE_POOL_H */ diff --git a/drivers/dma-buf/heaps/qcom_secure_system_heap.c b/drivers/dma-buf/heaps/qcom_secure_system_heap.c new file mode 100644 index 000000000000..3a96de78b338 --- /dev/null +++ b/drivers/dma-buf/heaps/qcom_secure_system_heap.c @@ -0,0 +1,716 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DMABUF System heap exporter + * Originally copied from: drivers/dma-buf/heaps/system_heap.c as of commit + * 263e38f82cbb ("dma-buf: heaps: Remove redundant heap identifier from system + * heap name") + * + * Additions taken from modifications to drivers/dma-buf/heaps/system-heap.c, + * from patches submitted, are listed below: + * + * Addition that modifies dma_buf ops to use SG tables taken from + * drivers/dma-buf/heaps/system-heap.c in: + * https://lore.kernel.org/lkml/20201017013255.43568-2-john.stultz@linaro.org/ + * + * Addition that skips unneeded syncs in the dma_buf ops taken from + * https://lore.kernel.org/lkml/20201017013255.43568-5-john.stultz@linaro.org/ + * + * Addition that tries to allocate higher order pages taken from + * https://lore.kernel.org/lkml/20201017013255.43568-6-john.stultz@linaro.org/ + * + * Addition that implements an uncached heap taken from + * https://lore.kernel.org/lkml/20201017013255.43568-8-john.stultz@linaro.org/, + * with our own modificaitons made to account for core kernel changes that are + * a part of the patch series. + * + * Pooling functionality taken from: + * Git-repo: https://git.linaro.org/people/john.stultz/android-dev.git + * Branch: dma-buf-heap-perf + * Git-commit: 6f080eb67dce63c6efa57ef564ca4cd762ccebb0 + * Git-commit: 6fb9593b928c4cb485bef4e88c59c6b9fdf11352 + * + * Branched off from qcom_system_heap.c as of commit a4e135a8e482 + * ("dt-bindings: ipcc: Add WPSS client to IPCC header") to accommodate secure + * pooling. + * + * Copyright (C) 2011 Google, Inc. + * Copyright (C) 2019, 2020 Linaro Ltd. + * + * Portions based off of Andrew Davis' SRAM heap: + * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/ + * Andrew F. Davis + * + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "qcom_dma_heap_secure_utils.h" +#include "qcom_dynamic_page_pool.h" +#include "qcom_sg_ops.h" +#include "qcom_secure_system_heap.h" +#include "qcom_system_heap.h" + +#define MAX_NR_PREFETCH_REGIONS 32 + +/* + * The video client may not hold the last reference count on the + * ion_buffer(s). Delay for a short time after the video client sends + * the IOC_DRAIN event to increase the chance that the reference + * count drops to zero. Time in milliseconds. + */ +#define SHRINK_DELAY 1000 + +struct prefetch_info { + struct list_head list; + struct dma_heap *heap; + u64 size; + bool shrink; +}; + +static LIST_HEAD(prefetch_list); +static DEFINE_SPINLOCK(work_lock); +static struct delayed_work prefetch_work; +static struct workqueue_struct *prefetch_wq; + +static LIST_HEAD(secure_heaps); + +static inline struct dynamic_page_pool **get_sys_heap_page_pool(void) +{ + static struct dynamic_page_pool **qcom_sys_heap_pools; + struct dma_heap *dma_heap; + struct qcom_system_heap *sys_heap; + + if (qcom_sys_heap_pools) + return qcom_sys_heap_pools; + + dma_heap = dma_heap_find("qcom,system"); + if (!dma_heap) { + pr_err("Unable to find the system heap\n"); + goto out; + } + + sys_heap = (struct qcom_system_heap *) dma_heap_get_drvdata(dma_heap); + qcom_sys_heap_pools = sys_heap->pool_list; + +out: + return qcom_sys_heap_pools; +} + +static struct sg_table *get_pages(u64 size, struct dma_heap *heap) +{ + LIST_HEAD(pages); + struct dynamic_page_pool **qcom_sys_heap_pools; + unsigned long size_remaining = size; + struct page *page, *tmp_page; + int max_order = orders[0]; + int num_pages = 0; + struct sg_table *sgt; + struct scatterlist *sg; + int ret; + + qcom_sys_heap_pools = get_sys_heap_page_pool(); + if (!qcom_sys_heap_pools) { + pr_err("%s: Couldn't obtain the pools for the system heap!\n", __func__); + return ERR_PTR(-EINVAL); + } + + while (size_remaining > 0) { + page = qcom_sys_heap_alloc_largest_available(qcom_sys_heap_pools, + size_remaining, + max_order); + + if (!page) { + pr_err("%s: Failed to get pages from the system heap: %d, %d!\n", + __func__, size_remaining, size); + ret = -ENOMEM; + goto err; + } + + list_add_tail(&page->lru, &pages); + size_remaining -= page_size(page); + max_order = compound_order(page); + num_pages++; + } + + sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); + if (!sgt) { + ret = -ENOMEM; + goto err; + } + + ret = sg_alloc_table(sgt, num_pages, GFP_KERNEL); + if (ret) { + pr_err("%s: sg table allocation failed with %d\n", __func__, ret); + goto free_table; + } + + sg = sgt->sgl; + list_for_each_entry(page, &pages, lru) { + sg_set_page(sg, page, page_size(page), 0); + sg = sg_next(sg); + } + + dma_map_sgtable(dma_heap_get_dev(heap), sgt, DMA_BIDIRECTIONAL, 0); + dma_unmap_sgtable(dma_heap_get_dev(heap), sgt, DMA_BIDIRECTIONAL, 0); + + return sgt; + +free_table: + kfree(sgt); +err: + list_for_each_entry_safe(page, tmp_page, &pages, lru) + __free_pages(page, compound_order(page)); + + return ERR_PTR(ret); +} + +static void process_one_prefetch(struct prefetch_info *info) +{ + struct qcom_secure_system_heap *secure_heap = dma_heap_get_drvdata(info->heap); + + struct sg_table *sgt; + struct scatterlist *sg; + struct page *page; + int ret; + int i, j; + + sgt = get_pages(info->size, info->heap); + if (IS_ERR(sgt)) + return; + + ret = hyp_assign_sg_from_flags(sgt, secure_heap->vmid, true); + if (ret) + goto err; + + for_each_sgtable_sg(sgt, sg, i) { + page = sg_page(sg); + + for (j = 0; j < NUM_ORDERS; j++) { + if (compound_order(page) == orders[j]) + break; + } + dynamic_page_pool_free(secure_heap->pool_list[j], page); + } + + sg_free_table(sgt); + + return; + +err: + for_each_sgtable_sg(sgt, sg, i) { + page = sg_page(sg); + __free_pages(page, compound_order(page)); + } + sg_free_table(sgt); +} + +static void process_one_shrink(struct prefetch_info *info) +{ + struct qcom_secure_system_heap *secure_heap = dma_heap_get_drvdata(info->heap); + + dynamic_page_pool_shrink_high_and_low(secure_heap->pool_list, + NUM_ORDERS, info->size / PAGE_SIZE); +} + +static void secure_system_heap_prefetch_work(struct work_struct *work) +{ + struct prefetch_info *info, *tmp; + unsigned long flags; + + spin_lock_irqsave(&work_lock, flags); + list_for_each_entry_safe(info, tmp, &prefetch_list, list) { + list_del(&info->list); + spin_unlock_irqrestore(&work_lock, flags); + + if (info->shrink) + process_one_shrink(info); + else + process_one_prefetch(info); + + kfree(info); + spin_lock_irqsave(&work_lock, flags); + } + spin_unlock_irqrestore(&work_lock, flags); +} + +static int alloc_prefetch_info(struct dma_buf_heap_prefetch_region *region, + bool shrink, struct list_head *items) +{ + struct prefetch_info *info; + + info = kmalloc(sizeof(*info), GFP_KERNEL); + if (!info) + return -ENOMEM; + + info->size = PAGE_ALIGN(region->size); + info->heap = region->heap; + info->shrink = shrink; + INIT_LIST_HEAD(&info->list); + list_add_tail(&info->list, items); + return 0; +} + +static int __qcom_secure_system_heap_resize(struct dma_buf_heap_prefetch_region *regions, + size_t nr_regions, + bool shrink) +{ + struct qcom_secure_system_heap *list_ptr, *secure_heap = NULL; + int i, ret = 0; + struct prefetch_info *info, *tmp; + unsigned long flags; + LIST_HEAD(items); + + if (!regions || nr_regions > MAX_NR_PREFETCH_REGIONS) { + pr_err("%s: Invalid input for %s\n", __func__, + (shrink) ? "drain" : "prefetch"); + return -EINVAL; + } + + for (i = 0; i < nr_regions; i++) { + list_for_each_entry(list_ptr, &secure_heaps, list) { + if (list_ptr == dma_heap_get_drvdata(regions->heap)) { + secure_heap = list_ptr; + break; + } + } + + if (!secure_heap) { + pr_err("%s: %s is not a secure system heap!\n", __func__, + dma_heap_get_name(regions->heap)); + ret = -EINVAL; + goto out_free; + } + + ret = alloc_prefetch_info(®ions[i], shrink, &items); + if (ret) + goto out_free; + } + + spin_lock_irqsave(&work_lock, flags); + list_splice_tail_init(&items, &prefetch_list); + queue_delayed_work(prefetch_wq, &prefetch_work, + shrink ? msecs_to_jiffies(SHRINK_DELAY) : 0); + spin_unlock_irqrestore(&work_lock, flags); + + return 0; + +out_free: + list_for_each_entry_safe(info, tmp, &items, list) { + list_del(&info->list); + kfree(info); + } + return ret; +} + +int qcom_secure_system_heap_prefetch(struct dma_buf_heap_prefetch_region *regions, + size_t nr_regions) +{ + return __qcom_secure_system_heap_resize(regions, nr_regions, false); +} +EXPORT_SYMBOL(qcom_secure_system_heap_prefetch); + +int qcom_secure_system_heap_drain(struct dma_buf_heap_prefetch_region *regions, + size_t nr_regions) +{ + return __qcom_secure_system_heap_resize(regions, nr_regions, true); +} +EXPORT_SYMBOL(qcom_secure_system_heap_drain); + +static enum dynamic_pool_callback_ret free_secure_pages(struct dynamic_page_pool *pool, + struct list_head *pages, + int num_pages) +{ + struct page *page; + struct sg_table sgt; + struct scatterlist *sg; + int ret; + + if (!num_pages) + return DYNAMIC_POOL_SUCCESS; + + ret = sg_alloc_table(&sgt, num_pages, GFP_KERNEL); + if (ret) + return DYNAMIC_POOL_FAILURE; + + sg = sgt.sgl; + list_for_each_entry(page, pages, lru) { + sg_set_page(sg, page, page_size(page), 0); + sg = sg_next(sg); + } + + ret = hyp_unassign_sg_from_flags(&sgt, pool->vmid, true); + sg_free_table(&sgt); + + if (ret) + return DYNAMIC_POOL_FAILURE; + + return DYNAMIC_POOL_SUCCESS; +} + +static void page_list_merge(struct list_head *secure_pages, + struct list_head *non_secure_pages, + struct scatterlist *sg, + struct scatterlist *non_secure_sg) +{ + bool is_secure_head, is_non_secure_head; + struct page *secure_page = list_first_entry(secure_pages, struct page, lru); + struct page *non_secure_page = list_first_entry(non_secure_pages, struct page, lru); + + do { + is_secure_head = list_entry_is_head(secure_page, secure_pages, lru); + is_non_secure_head = list_entry_is_head(non_secure_page, non_secure_pages, lru); + + if (!is_secure_head && !is_non_secure_head) { + if (page_size(non_secure_page) >= page_size(secure_page)) { + sg_set_page(sg, non_secure_page, + page_size(non_secure_page), 0); + sg_set_page(non_secure_sg, non_secure_page, + page_size(non_secure_page), 0); + + non_secure_page = list_next_entry(non_secure_page, lru); + non_secure_sg = sg_next(non_secure_sg); + } else { + sg_set_page(sg, secure_page, + page_size(secure_page), 0); + secure_page = list_next_entry(secure_page, lru); + } + } else if (!is_non_secure_head) { + sg_set_page(sg, non_secure_page, + page_size(non_secure_page), 0); + sg_set_page(non_secure_sg, non_secure_page, + page_size(non_secure_page), 0); + + non_secure_page = list_next_entry(non_secure_page, lru); + non_secure_sg = sg_next(non_secure_sg); + } else if (!is_secure_head) { + sg_set_page(sg, secure_page, + page_size(secure_page), 0); + secure_page = list_next_entry(secure_page, lru); + } + + sg = sg_next(sg); + } while (sg); +} + +static int page_to_pool_ind(struct page *page) +{ + int i; + + for (i = 0; i < NUM_ORDERS; i++) + if (compound_order(page) == orders[i]) + break; + return i; +} + +static void system_heap_free(struct qcom_sg_buffer *buffer) +{ + struct qcom_secure_system_heap *sys_heap; + struct sg_table *table; + struct scatterlist *sg; + int i, j; + + sys_heap = dma_heap_get_drvdata(buffer->heap); + table = &buffer->sg_table; + + for_each_sg(table->sgl, sg, table->nents, i) { + struct page *page = sg_page(sg); + + for (j = 0; j < NUM_ORDERS; j++) { + if (compound_order(page) == orders[j]) + break; + } + dynamic_page_pool_free(sys_heap->pool_list[j], page); + } + sg_free_table(table); + kfree(buffer); +} + +static struct page *alloc_largest_available(struct dynamic_page_pool **pools, + unsigned long size, + unsigned int max_order, + bool *page_from_secure_pool) +{ + struct page *page = NULL; + int i; + struct dynamic_page_pool **qcom_sys_heap_pools = get_sys_heap_page_pool(); + + if (!qcom_sys_heap_pools) { + pr_err("%s: Couldn't obtain the pools for the system heap!\n", __func__); + return NULL; + } + + *page_from_secure_pool = true; + + for (i = 0; i < NUM_ORDERS; i++) { + if (size < (PAGE_SIZE << orders[i])) + continue; + if (max_order < orders[i]) + continue; + + mutex_lock(&pools[i]->mutex); + if (pools[i]->high_count) + page = dynamic_page_pool_remove(pools[i], true); + else if (pools[i]->low_count) + page = dynamic_page_pool_remove(pools[i], false); + mutex_unlock(&pools[i]->mutex); + + if (!page) + continue; + return page; + } + + *page_from_secure_pool = false; + + return qcom_sys_heap_alloc_largest_available(qcom_sys_heap_pools, + size, + max_order); +} + +static struct dma_buf *system_heap_allocate(struct dma_heap *heap, + unsigned long len, + unsigned long fd_flags, + unsigned long heap_flags) +{ + struct qcom_secure_system_heap *sys_heap; + struct qcom_sg_buffer *buffer; + DEFINE_DMA_BUF_EXPORT_INFO(exp_info); + unsigned long size_remaining = len; + unsigned int max_order = orders[0]; + bool page_from_secure_pool; + struct dma_buf *dmabuf; + struct sg_table *table, non_secure_table; + struct scatterlist *sg, *non_secure_sg = NULL; + LIST_HEAD(secure_pages); + LIST_HEAD(non_secure_pages); + struct page *page, *tmp_page; + int total_pages = 0, num_non_secure_pages = 0; + int ret = -ENOMEM, hyp_ret = 0; + int perms; + int vmid; + + buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); + if (!buffer) + return ERR_PTR(-ENOMEM); + + sys_heap = dma_heap_get_drvdata(heap); + + INIT_LIST_HEAD(&buffer->attachments); + mutex_init(&buffer->lock); + buffer->heap = heap; + buffer->len = len; + buffer->uncached = true; + buffer->free = system_heap_free; + + while (size_remaining > 0) { + /* + * Avoid trying to allocate memory if the process + * has been killed by SIGKILL + */ + if (fatal_signal_pending(current)) + goto free_buffer; + + page = alloc_largest_available(sys_heap->pool_list, + size_remaining, + max_order, + &page_from_secure_pool); + if (!page) + goto free_buffer; + + if (page_from_secure_pool) { + list_add_tail(&page->lru, &secure_pages); + } else { + num_non_secure_pages++; + list_add_tail(&page->lru, &non_secure_pages); + } + + size_remaining -= page_size(page); + max_order = compound_order(page); + total_pages++; + } + + table = &buffer->sg_table; + if (sg_alloc_table(table, total_pages, GFP_KERNEL)) + goto free_buffer; + sg = table->sgl; + + if (num_non_secure_pages) { + if (sg_alloc_table(&non_secure_table, num_non_secure_pages, GFP_KERNEL)) + goto free_sg; + non_secure_sg = non_secure_table.sgl; + } + + page_list_merge(&secure_pages, &non_secure_pages, sg, non_secure_sg); + + if (num_non_secure_pages) { + dma_map_sgtable(dma_heap_get_dev(heap), &non_secure_table, DMA_BIDIRECTIONAL, 0); + dma_unmap_sgtable(dma_heap_get_dev(heap), &non_secure_table, DMA_BIDIRECTIONAL, 0); + + hyp_ret = hyp_assign_sg_from_flags(&non_secure_table, sys_heap->vmid, true); + if (hyp_ret) + goto free_non_secure_sg; + } + + perms = msm_secure_get_vmid_perms(sys_heap->vmid); + vmid = get_secure_vmid(sys_heap->vmid); + buffer->vmperm = mem_buf_vmperm_alloc_staticvm(table, + &vmid, &perms, 1); + + if (IS_ERR(buffer->vmperm)) { + ret = PTR_ERR(buffer->vmperm); + goto hyp_unassign; + } + + /* create the dmabuf */ + exp_info.exp_name = dma_heap_get_name(heap); + exp_info.ops = &qcom_sg_buf_ops.dma_ops; + exp_info.size = buffer->len; + exp_info.flags = fd_flags; + exp_info.priv = buffer; + dmabuf = mem_buf_dma_buf_export(&exp_info, &qcom_sg_buf_ops); + if (IS_ERR(dmabuf)) { + ret = PTR_ERR(dmabuf); + goto vmperm_release; + } + + if (num_non_secure_pages) + sg_free_table(&non_secure_table); + + return dmabuf; + +vmperm_release: + mem_buf_vmperm_release(buffer->vmperm); + +hyp_unassign: + /* We check PagePrivate() below to see if we've reclaimed a particular page */ + hyp_ret = hyp_unassign_sg_from_flags(&non_secure_table, sys_heap->vmid, true); + +free_non_secure_sg: + if (num_non_secure_pages) + sg_free_table(&non_secure_table); +free_sg: + sg_free_table(table); +free_buffer: + list_for_each_entry_safe(page, tmp_page, &secure_pages, lru) + dynamic_page_pool_free(sys_heap->pool_list[page_to_pool_ind(page)], page); + if (!hyp_ret) + list_for_each_entry_safe(page, tmp_page, &non_secure_pages, lru) + __free_pages(page, compound_order(page)); + + kfree(buffer); + + return ERR_PTR(ret); +} + + +static const struct dma_heap_ops system_heap_ops = { + .allocate = system_heap_allocate, +}; + +static int create_prefetch_workqueue(void) +{ + static bool registered; + + if (registered) + return 0; + + INIT_DELAYED_WORK(&prefetch_work, + secure_system_heap_prefetch_work); + + prefetch_wq = alloc_workqueue("system_secure_prefetch_wq", + WQ_UNBOUND | WQ_FREEZABLE, 0); + if (!prefetch_wq) { + pr_err("Failed to create system secure prefetch workqueue\n"); + return -ENOMEM; + } + + registered = true; + return 0; +} + +void qcom_secure_system_heap_create(const char *name, const char *secure_system_alias, int vmid) +{ + struct dma_heap_export_info exp_info; + struct dma_heap *heap; + struct qcom_secure_system_heap *sys_heap; + int ret; + + if (get_secure_vmid(vmid) == -EINVAL || hweight_long(vmid) != 1) { + pr_err("Invalid VMID or supplied more than one VMID\n"); + ret = -EINVAL; + goto out; + } + + ret = dynamic_page_pool_init_shrinker(); + if (ret) + goto out; + + ret = create_prefetch_workqueue(); + if (ret) + goto out; + + sys_heap = kzalloc(sizeof(*sys_heap), GFP_KERNEL); + if (!sys_heap) { + ret = -ENOMEM; + goto out; + } + + sys_heap->vmid = vmid; + sys_heap->pool_list = dynamic_page_pool_create_pools(vmid, free_secure_pages); + if (IS_ERR(sys_heap->pool_list)) { + ret = PTR_ERR(sys_heap->pool_list); + goto free_heap; + } + + exp_info.name = name; + exp_info.ops = &system_heap_ops; + exp_info.priv = sys_heap; + + heap = dma_heap_add(&exp_info); + if (IS_ERR(heap)) { + ret = PTR_ERR(heap); + goto free_pools; + } + + dma_coerce_mask_and_coherent(dma_heap_get_dev(heap), DMA_BIT_MASK(64)); + + list_add(&sys_heap->list, &secure_heaps); + + pr_info("%s: DMA-BUF Heap: Created '%s'\n", __func__, name); + + if (secure_system_alias != NULL) { + exp_info.name = secure_system_alias; + + heap = dma_heap_add(&exp_info); + if (IS_ERR(heap)) { + pr_err("%s: Failed to create '%s', error is %d\n", __func__, + secure_system_alias, PTR_ERR(heap)); + return; + } + + dma_coerce_mask_and_coherent(dma_heap_get_dev(heap), DMA_BIT_MASK(64)); + + pr_info("%s: DMA-BUF Heap: Created '%s'\n", __func__, secure_system_alias); + } + + return; + +free_pools: + dynamic_page_pool_release_pools(sys_heap->pool_list); + +free_heap: + kfree(sys_heap); + +out: + pr_err("%s: Failed to create '%s', error is %d\n", __func__, name, ret); +} diff --git a/drivers/dma-buf/heaps/qcom_secure_system_heap.h b/drivers/dma-buf/heaps/qcom_secure_system_heap.h new file mode 100644 index 000000000000..93c8154acf88 --- /dev/null +++ b/drivers/dma-buf/heaps/qcom_secure_system_heap.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#ifndef _QCOM_SECURE_SYSTEM_HEAP_H +#define _QCOM_SECURE_SYSTEM_HEAP_H + +#include +#include +#include "qcom_dynamic_page_pool.h" + +struct qcom_secure_system_heap { + struct dynamic_page_pool **pool_list; + int vmid; + struct list_head list; +}; + +#ifdef CONFIG_QCOM_DMABUF_HEAPS_SYSTEM_SECURE +void qcom_secure_system_heap_create(const char *name, const char *secure_system_alias, + int vmid); +#else +static void qcom_secure_system_heap_create(const char *name, const char *secure_system_alias, + int vmid) +{ + +} +#endif + +#endif /* _QCOM_SECURE_SYSTEM_HEAP_H */ diff --git a/drivers/dma-buf/heaps/qcom_sg_ops.c b/drivers/dma-buf/heaps/qcom_sg_ops.c new file mode 100644 index 000000000000..dee93ec563fb --- /dev/null +++ b/drivers/dma-buf/heaps/qcom_sg_ops.c @@ -0,0 +1,544 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * mmap() algorithm taken from drivers/staging/android/ion/ion_heap.c as + * of commit a3ec289e74b4 ("arm-smmu: Fix missing qsmmuv500 callback") + * + * Copyright (C) 2011 Google, Inc. + * Copyright (C) 2019, 2020 Linaro Ltd. + * + * Portions based off of Andrew Davis' SRAM heap: + * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/ + * Andrew F. Davis + * + * These ops were base of the ops in drivers/dma-buf/heaps/system-heap.c from + * https://lore.kernel.org/lkml/20201017013255.43568-2-john.stultz@linaro.org/ + * + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "qcom_sg_ops.h" + +static struct sg_table *dup_sg_table(struct sg_table *table) +{ + struct sg_table *new_table; + int ret, i; + struct scatterlist *sg, *new_sg; + + new_table = kzalloc(sizeof(*new_table), GFP_KERNEL); + if (!new_table) + return ERR_PTR(-ENOMEM); + + ret = sg_alloc_table(new_table, table->orig_nents, GFP_KERNEL); + if (ret) { + kfree(new_table); + return ERR_PTR(-ENOMEM); + } + + new_sg = new_table->sgl; + for_each_sgtable_sg(table, sg, i) { + sg_set_page(new_sg, sg_page(sg), sg->length, sg->offset); + new_sg = sg_next(new_sg); + } + + return new_table; +} + +static int qcom_sg_attach(struct dma_buf *dmabuf, + struct dma_buf_attachment *attachment) +{ + struct qcom_sg_buffer *buffer = dmabuf->priv; + struct dma_heap_attachment *a; + struct sg_table *table; + + a = kzalloc(sizeof(*a), GFP_KERNEL); + if (!a) + return -ENOMEM; + + table = dup_sg_table(&buffer->sg_table); + if (IS_ERR(table)) { + kfree(a); + return -ENOMEM; + } + + a->table = table; + a->dev = attachment->dev; + INIT_LIST_HEAD(&a->list); + a->mapped = false; + + attachment->priv = a; + + mutex_lock(&buffer->lock); + list_add(&a->list, &buffer->attachments); + mutex_unlock(&buffer->lock); + + return 0; +} + +static void qcom_sg_detach(struct dma_buf *dmabuf, + struct dma_buf_attachment *attachment) +{ + struct qcom_sg_buffer *buffer = dmabuf->priv; + struct dma_heap_attachment *a = attachment->priv; + + mutex_lock(&buffer->lock); + list_del(&a->list); + mutex_unlock(&buffer->lock); + + sg_free_table(a->table); + kfree(a->table); + kfree(a); +} + +static struct sg_table *qcom_sg_map_dma_buf(struct dma_buf_attachment *attachment, + enum dma_data_direction direction) +{ + struct dma_heap_attachment *a = attachment->priv; + struct sg_table *table = a->table; + struct qcom_sg_buffer *buffer; + struct mem_buf_vmperm *vmperm; + unsigned long attrs = attachment->dma_map_attrs; + int ret; + + buffer = attachment->dmabuf->priv; + vmperm = buffer->vmperm; + + /* Prevent map/unmap during begin/end_cpu_access */ + mutex_lock(&buffer->lock); + + /* Ensure VM permissions are constant while the buffer is mapped */ + mem_buf_vmperm_pin(vmperm); + if (buffer->uncached || !mem_buf_vmperm_can_cmo(vmperm)) + attrs |= DMA_ATTR_SKIP_CPU_SYNC; + + if (attrs & DMA_ATTR_DELAYED_UNMAP) + ret = msm_dma_map_sgtable(attachment->dev, table, direction, + attachment->dmabuf, attrs); + else + ret = dma_map_sgtable(attachment->dev, table, direction, attrs); + + if (ret) { + table = ERR_PTR(-ENOMEM); + goto err_map_sgtable; + } + + a->mapped = true; + mutex_unlock(&buffer->lock); + return table; + +err_map_sgtable: + mem_buf_vmperm_unpin(vmperm); + mutex_unlock(&buffer->lock); + return table; +} + +static void qcom_sg_unmap_dma_buf(struct dma_buf_attachment *attachment, + struct sg_table *table, + enum dma_data_direction direction) +{ + struct dma_heap_attachment *a = attachment->priv; + struct qcom_sg_buffer *buffer; + struct mem_buf_vmperm *vmperm; + unsigned long attrs = attachment->dma_map_attrs; + + buffer = attachment->dmabuf->priv; + vmperm = buffer->vmperm; + + /* Prevent map/unmap during begin/end_cpu_access */ + mutex_lock(&buffer->lock); + + if (buffer->uncached || !mem_buf_vmperm_can_cmo(vmperm)) + attrs |= DMA_ATTR_SKIP_CPU_SYNC; + + a->mapped = false; + if (attrs & DMA_ATTR_DELAYED_UNMAP) + msm_dma_unmap_sgtable(attachment->dev, table, direction, + attachment->dmabuf, attrs); + else + dma_unmap_sgtable(attachment->dev, table, direction, attrs); + mem_buf_vmperm_unpin(vmperm); + mutex_unlock(&buffer->lock); +} + +static int qcom_sg_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, + enum dma_data_direction direction) +{ + struct qcom_sg_buffer *buffer = dmabuf->priv; + struct dma_heap_attachment *a; + + if (buffer->uncached) + return 0; + + mutex_lock(&buffer->lock); + + /* Keep the same behavior as ion by returning 0 instead of -EPERM */ + if (!mem_buf_vmperm_can_cmo(buffer->vmperm)) { + mutex_unlock(&buffer->lock); + return 0; + } + + if (buffer->vmap_cnt) + invalidate_kernel_vmap_range(buffer->vaddr, buffer->len); + + list_for_each_entry(a, &buffer->attachments, list) { + if (!a->mapped) + continue; + dma_sync_sgtable_for_cpu(a->dev, a->table, direction); + } + + mutex_unlock(&buffer->lock); + + return 0; +} + +static int qcom_sg_dma_buf_end_cpu_access(struct dma_buf *dmabuf, + enum dma_data_direction direction) +{ + struct qcom_sg_buffer *buffer = dmabuf->priv; + struct dma_heap_attachment *a; + + if (buffer->uncached) + return 0; + + mutex_lock(&buffer->lock); + + /* Keep the same behavior as ion by returning 0 instead of -EPERM */ + if (!mem_buf_vmperm_can_cmo(buffer->vmperm)) { + mutex_unlock(&buffer->lock); + return 0; + } + + if (buffer->vmap_cnt) + flush_kernel_vmap_range(buffer->vaddr, buffer->len); + + list_for_each_entry(a, &buffer->attachments, list) { + if (!a->mapped) + continue; + dma_sync_sgtable_for_device(a->dev, a->table, direction); + } + mutex_unlock(&buffer->lock); + + return 0; +} + +static int sgl_sync_range(struct device *dev, struct scatterlist *sgl, + unsigned int nents, unsigned long offset, + unsigned long length, + enum dma_data_direction dir, bool for_cpu) +{ + int i; + struct scatterlist *sg; + unsigned int len = 0; + dma_addr_t sg_dma_addr; + + for_each_sg(sgl, sg, nents, i) { + if (sg_dma_len(sg) == 0) + break; + + if (i > 0) { + pr_warn_ratelimited("Partial cmo only supported with 1 segment\n" + "is dma_set_max_seg_size being set on dev:%s\n", + dev_name(dev)); + return -EINVAL; + } + } + + for_each_sg(sgl, sg, nents, i) { + unsigned int sg_offset, sg_left, size = 0; + + if (i == 0) + sg_dma_addr = sg_dma_address(sg); + + len += sg->length; + if (len <= offset) { + sg_dma_addr += sg->length; + continue; + } + + sg_left = len - offset; + sg_offset = sg->length - sg_left; + + size = (length < sg_left) ? length : sg_left; + if (for_cpu) + dma_sync_single_range_for_cpu(dev, sg_dma_addr, + sg_offset, size, dir); + else + dma_sync_single_range_for_device(dev, sg_dma_addr, + sg_offset, size, dir); + + offset += size; + length -= size; + sg_dma_addr += sg->length; + + if (length == 0) + break; + } + + return 0; +} + +static int qcom_sg_dma_buf_begin_cpu_access_partial(struct dma_buf *dmabuf, + enum dma_data_direction dir, + unsigned int offset, + unsigned int len) +{ + struct qcom_sg_buffer *buffer = dmabuf->priv; + struct dma_heap_attachment *a; + int ret = 0; + + if (buffer->uncached) + return 0; + + mutex_lock(&buffer->lock); + + /* Keep the same behavior as ion by returning 0 instead of -EPERM */ + if (!mem_buf_vmperm_can_cmo(buffer->vmperm)) { + mutex_unlock(&buffer->lock); + return 0; + } + + if (buffer->vmap_cnt) + invalidate_kernel_vmap_range(buffer->vaddr + offset, len); + + list_for_each_entry(a, &buffer->attachments, list) { + if (!a->mapped) + continue; + + ret = sgl_sync_range(a->dev, a->table->sgl, a->table->orig_nents, + offset, len, dir, true); + } + mutex_unlock(&buffer->lock); + + return ret; +} + +static int qcom_sg_dma_buf_end_cpu_access_partial(struct dma_buf *dmabuf, + enum dma_data_direction direction, + unsigned int offset, + unsigned int len) +{ + struct qcom_sg_buffer *buffer = dmabuf->priv; + struct dma_heap_attachment *a; + int ret = 0; + + if (buffer->uncached) + return 0; + + mutex_lock(&buffer->lock); + + /* Keep the same behavior as ion by returning 0 instead of -EPERM */ + if (!mem_buf_vmperm_can_cmo(buffer->vmperm)) { + mutex_unlock(&buffer->lock); + return 0; + } + + if (buffer->vmap_cnt) + flush_kernel_vmap_range(buffer->vaddr + offset, len); + + list_for_each_entry(a, &buffer->attachments, list) { + if (!a->mapped) + continue; + + ret = sgl_sync_range(a->dev, a->table->sgl, a->table->orig_nents, + offset, len, direction, false); + } + mutex_unlock(&buffer->lock); + + return ret; +} + +static void qcom_sg_vm_ops_open(struct vm_area_struct *vma) +{ + struct mem_buf_vmperm *vmperm = vma->vm_private_data; + + mem_buf_vmperm_pin(vmperm); +} + +static void qcom_sg_vm_ops_close(struct vm_area_struct *vma) +{ + struct mem_buf_vmperm *vmperm = vma->vm_private_data; + + mem_buf_vmperm_unpin(vmperm); +} + +static const struct vm_operations_struct qcom_sg_vm_ops = { + .open = qcom_sg_vm_ops_open, + .close = qcom_sg_vm_ops_close, +}; + +static int qcom_sg_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) +{ + struct qcom_sg_buffer *buffer = dmabuf->priv; + struct sg_table *table = &buffer->sg_table; + struct scatterlist *sg; + unsigned long addr = vma->vm_start; + unsigned long offset = vma->vm_pgoff * PAGE_SIZE; + int ret; + int i; + + mem_buf_vmperm_pin(buffer->vmperm); + if (!mem_buf_vmperm_can_mmap(buffer->vmperm, vma)) { + mem_buf_vmperm_unpin(buffer->vmperm); + return -EPERM; + } + + vma->vm_ops = &qcom_sg_vm_ops; + vma->vm_private_data = buffer->vmperm; + if (buffer->uncached) + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); + + for_each_sg(table->sgl, sg, table->nents, i) { + struct page *page = sg_page(sg); + unsigned long remainder = vma->vm_end - addr; + unsigned long len = sg->length; + + if (offset >= sg->length) { + offset -= sg->length; + continue; + } else if (offset) { + page += offset / PAGE_SIZE; + len = sg->length - offset; + offset = 0; + } + len = min(len, remainder); + ret = remap_pfn_range(vma, addr, page_to_pfn(page), len, + vma->vm_page_prot); + if (ret) { + mem_buf_vmperm_unpin(buffer->vmperm); + return ret; + } + addr += len; + if (addr >= vma->vm_end) + return 0; + } + return 0; +} + +static void *qcom_sg_do_vmap(struct qcom_sg_buffer *buffer) +{ + struct sg_table *table = &buffer->sg_table; + int npages = PAGE_ALIGN(buffer->len) / PAGE_SIZE; + struct page **pages = vmalloc(sizeof(struct page *) * npages); + struct page **tmp = pages; + struct sg_page_iter piter; + pgprot_t pgprot = PAGE_KERNEL; + void *vaddr; + + if (!pages) + return ERR_PTR(-ENOMEM); + + if (buffer->uncached) + pgprot = pgprot_writecombine(PAGE_KERNEL); + + for_each_sgtable_page(table, &piter, 0) { + WARN_ON(tmp - pages >= npages); + *tmp++ = sg_page_iter_page(&piter); + } + + vaddr = vmap(pages, npages, VM_MAP, pgprot); + vfree(pages); + + if (!vaddr) + return ERR_PTR(-ENOMEM); + + return vaddr; +} + +static int qcom_sg_vmap(struct dma_buf *dmabuf, struct dma_buf_map *map) +{ + struct qcom_sg_buffer *buffer = dmabuf->priv; + void *vaddr; + int ret = 0; + + mem_buf_vmperm_pin(buffer->vmperm); + if (!mem_buf_vmperm_can_vmap(buffer->vmperm)) { + mem_buf_vmperm_unpin(buffer->vmperm); + return -EPERM; + } + + mutex_lock(&buffer->lock); + if (buffer->vmap_cnt) { + buffer->vmap_cnt++; + dma_buf_map_set_vaddr(map, buffer->vaddr); + goto out; + } + + vaddr = qcom_sg_do_vmap(buffer); + if (IS_ERR(vaddr)) { + ret = PTR_ERR(vaddr); + mem_buf_vmperm_unpin(buffer->vmperm); + goto out; + } + + buffer->vaddr = vaddr; + buffer->vmap_cnt++; + dma_buf_map_set_vaddr(map, buffer->vaddr); +out: + mutex_unlock(&buffer->lock); + + return ret; +} + +static void qcom_sg_vunmap(struct dma_buf *dmabuf, struct dma_buf_map *map) +{ + struct qcom_sg_buffer *buffer = dmabuf->priv; + + mutex_lock(&buffer->lock); + if (!--buffer->vmap_cnt) { + vunmap(buffer->vaddr); + buffer->vaddr = NULL; + } + mem_buf_vmperm_unpin(buffer->vmperm); + mutex_unlock(&buffer->lock); + dma_buf_map_clear(map); +} + +static void qcom_sg_release(struct dma_buf *dmabuf) +{ + struct qcom_sg_buffer *buffer = dmabuf->priv; + + if (mem_buf_vmperm_release(buffer->vmperm)) + return; + + msm_dma_buf_freed(buffer); + buffer->free(buffer); +} + +static struct mem_buf_vmperm *qcom_sg_lookup_vmperm(struct dma_buf *dmabuf) +{ + struct qcom_sg_buffer *buffer = dmabuf->priv; + + return buffer->vmperm; +} + +struct mem_buf_dma_buf_ops qcom_sg_buf_ops = { + .attach = qcom_sg_attach, + .lookup = qcom_sg_lookup_vmperm, + .dma_ops = { + .attach = NULL, /* Will be set by mem_buf_dma_buf_export */ + .detach = qcom_sg_detach, + .map_dma_buf = qcom_sg_map_dma_buf, + .unmap_dma_buf = qcom_sg_unmap_dma_buf, + .begin_cpu_access = qcom_sg_dma_buf_begin_cpu_access, + .end_cpu_access = qcom_sg_dma_buf_end_cpu_access, + .begin_cpu_access_partial = qcom_sg_dma_buf_begin_cpu_access_partial, + .end_cpu_access_partial = qcom_sg_dma_buf_end_cpu_access_partial, + .mmap = qcom_sg_mmap, + .vmap = qcom_sg_vmap, + .vunmap = qcom_sg_vunmap, + .release = qcom_sg_release, + } +}; +EXPORT_SYMBOL(qcom_sg_buf_ops); diff --git a/drivers/dma-buf/heaps/qcom_sg_ops.h b/drivers/dma-buf/heaps/qcom_sg_ops.h new file mode 100644 index 000000000000..346095a08181 --- /dev/null +++ b/drivers/dma-buf/heaps/qcom_sg_ops.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2011 Google, Inc. + * Copyright (C) 2019, 2020 Linaro Ltd. + * + * Portions based off of Andrew Davis' SRAM heap: + * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/ + * Andrew F. Davis + * + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#ifndef _QCOM_SG_OPS_H +#define _QCOM_SG_OPS_H + +#include +#include +#include +#include + +struct qcom_sg_buffer { + struct dma_heap *heap; + struct list_head attachments; + struct mutex lock; + unsigned long len; + struct sg_table sg_table; + int vmap_cnt; + void *vaddr; + bool uncached; + struct mem_buf_vmperm *vmperm; + void (*free)(struct qcom_sg_buffer *buffer); +}; + +struct dma_heap_attachment { + struct device *dev; + struct sg_table *table; + struct list_head list; + bool mapped; +}; + +extern struct mem_buf_dma_buf_ops qcom_sg_buf_ops; + +#endif /* _QCOM_SG_OPS_H */ diff --git a/drivers/dma-buf/heaps/qcom_system_heap.c b/drivers/dma-buf/heaps/qcom_system_heap.c new file mode 100644 index 000000000000..1cc38ffc0029 --- /dev/null +++ b/drivers/dma-buf/heaps/qcom_system_heap.c @@ -0,0 +1,584 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DMABUF System heap exporter + * Originally copied from: drivers/dma-buf/heaps/system_heap.c as of commit + * 263e38f82cbb ("dma-buf: heaps: Remove redundant heap identifier from system + * heap name") + * + * Additions taken from modifications to drivers/dma-buf/heaps/system-heap.c, + * from patches submitted, are listed below: + * + * Addition that modifies dma_buf ops to use SG tables taken from + * drivers/dma-buf/heaps/system-heap.c in: + * https://lore.kernel.org/lkml/20201017013255.43568-2-john.stultz@linaro.org/ + * + * Addition that skips unneeded syncs in the dma_buf ops taken from + * https://lore.kernel.org/lkml/20201017013255.43568-5-john.stultz@linaro.org/ + * + * Addition that tries to allocate higher order pages taken from + * https://lore.kernel.org/lkml/20201017013255.43568-6-john.stultz@linaro.org/ + * + * Addition that implements an uncached heap taken from + * https://lore.kernel.org/lkml/20201017013255.43568-8-john.stultz@linaro.org/, + * with our own modificaitons made to account for core kernel changes that are + * a part of the patch series. + * + * Pooling functionality taken from: + * Git-repo: https://git.linaro.org/people/john.stultz/android-dev.git + * Branch: dma-buf-heap-perf + * Git-commit: 6f080eb67dce63c6efa57ef564ca4cd762ccebb0 + * Git-commit: 6fb9593b928c4cb485bef4e88c59c6b9fdf11352 + * + * Copyright (C) 2011 Google, Inc. + * Copyright (C) 2019, 2020 Linaro Ltd. + * + * Portions based off of Andrew Davis' SRAM heap: + * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/ + * Andrew F. Davis + * + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "qcom_dma_heap_secure_utils.h" +#include "qcom_dynamic_page_pool.h" +#include "qcom_sg_ops.h" +#include "qcom_system_heap.h" + +#if IS_ENABLED(CONFIG_QCOM_DMABUF_HEAPS_PAGE_POOL_REFILL) +#define DYNAMIC_POOL_FILL_MARK (100 * SZ_1M) +#define DYNAMIC_POOL_LOW_MARK_PERCENT 40UL +#define DYNAMIC_POOL_LOW_MARK ((DYNAMIC_POOL_FILL_MARK * DYNAMIC_POOL_LOW_MARK_PERCENT) / 100) + +#define DYNAMIC_POOL_REFILL_DEFER_WINDOW_MS 10 +#define DYNAMIC_POOL_KTHREAD_NICE_VAL 10 + +static int get_dynamic_pool_fillmark(struct dynamic_page_pool *pool) +{ + return DYNAMIC_POOL_FILL_MARK / (PAGE_SIZE << pool->order); +} + +static bool dynamic_pool_fillmark_reached(struct dynamic_page_pool *pool) +{ + return atomic_read(&pool->count) >= get_dynamic_pool_fillmark(pool); +} + +static int get_dynamic_pool_lowmark(struct dynamic_page_pool *pool) +{ + return DYNAMIC_POOL_LOW_MARK / (PAGE_SIZE << pool->order); +} + +static bool dynamic_pool_count_below_lowmark(struct dynamic_page_pool *pool) +{ + return atomic_read(&pool->count) < get_dynamic_pool_lowmark(pool); +} + +/* Based on gfp_zone() in mm/mmzone.c since it is not exported. */ +static enum zone_type dynamic_pool_gfp_zone(gfp_t flags) +{ + enum zone_type z; + gfp_t local_flags = flags; + int bit; + + bit = (__force int) ((local_flags) & GFP_ZONEMASK); + + z = (GFP_ZONE_TABLE >> (bit * GFP_ZONES_SHIFT)) & + ((1 << GFP_ZONES_SHIFT) - 1); + VM_BUG_ON((GFP_ZONE_BAD >> bit) & 1); + return z; +} + +/* + * Based on __zone_watermark_ok() in mm/page_alloc.c since it is not exported. + * + * Return true if free base pages are above 'mark'. For high-order checks it + * will return true of the order-0 watermark is reached and there is at least + * one free page of a suitable size. Checking now avoids taking the zone lock + * to check in the allocation paths if no pages are free. + */ +static bool __dynamic_pool_zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, + int highest_zoneidx, long free_pages) +{ + long min = mark; + long unusable_free; + int o; + + /* + * Access to high atomic reserves is not required, and CMA should not be + * used, since these allocations are non-movable. + */ + unusable_free = ((1 << order) - 1) + z->nr_reserved_highatomic; +#ifdef CONFIG_CMA + unusable_free += zone_page_state(z, NR_FREE_CMA_PAGES); +#endif + + /* free_pages may go negative - that's OK */ + free_pages -= unusable_free; + + /* + * Check watermarks for an order-0 allocation request. If these + * are not met, then a high-order request also cannot go ahead + * even if a suitable page happened to be free. + * + * 'min' can be taken as 'mark' since we do not expect these allocations + * to require disruptive actions (such as running the OOM killer) or + * a lot of effort. + */ + if (free_pages <= min + z->lowmem_reserve[highest_zoneidx]) + return false; + + /* If this is an order-0 request then the watermark is fine */ + if (!order) + return true; + + /* For a high-order request, check at least one suitable page is free */ + for (o = order; o < MAX_ORDER; o++) { + struct free_area *area = &z->free_area[o]; + int mt; + + if (!area->nr_free) + continue; + + for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) { + if (!free_area_empty(area, mt)) + return true; + } + } + + return false; +} + +/* Based on zone_watermark_ok_safe from mm/page_alloc.c since it is not exported. */ +static bool dynamic_pool_zone_watermark_ok_safe(struct zone *z, unsigned int order, + unsigned long mark, int highest_zoneidx) +{ + long free_pages = zone_page_state(z, NR_FREE_PAGES); + + if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark) + free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES); + + return __dynamic_pool_zone_watermark_ok(z, order, mark, highest_zoneidx, free_pages); +} + +/* do a simple check to see if we are in any low memory situation */ +static bool dynamic_pool_refill_ok(struct dynamic_page_pool *pool) +{ + struct zone *zone; + int i, mark; + enum zone_type classzone_idx = dynamic_pool_gfp_zone(pool->gfp_mask); + s64 delta; + + /* check if we are within the refill defer window */ + delta = ktime_ms_delta(ktime_get(), pool->last_low_watermark_ktime); + if (delta < DYNAMIC_POOL_REFILL_DEFER_WINDOW_MS) + return false; + + /* + * make sure that if we allocate a pool->order page from buddy, + * we don't put the zone watermarks below the high threshold. + * This makes sure there's no unwanted repetitive refilling and + * reclaiming of buddy pages on the pool. + */ + for (i = classzone_idx; i >= 0; i--) { + zone = &NODE_DATA(numa_node_id())->node_zones[i]; + + if (!strcmp(zone->name, "DMA32")) + continue; + + mark = high_wmark_pages(zone); + mark += 1 << pool->order; + if (!dynamic_pool_zone_watermark_ok_safe(zone, pool->order, mark, classzone_idx)) { + pool->last_low_watermark_ktime = ktime_get(); + return false; + } + } + + return true; +} + +static void dynamic_page_pool_refill(struct dynamic_page_pool *pool) +{ + struct page *page; + gfp_t gfp_refill = (pool->gfp_mask | __GFP_RECLAIM) & ~__GFP_NORETRY; + + /* skip refilling order 0 pools */ + if (!pool->order) + return; + + while (!dynamic_pool_fillmark_reached(pool) && dynamic_pool_refill_ok(pool)) { + page = alloc_pages(gfp_refill, pool->order); + if (!page) + break; + + dynamic_page_pool_add(pool, page); + } +} + +static bool dynamic_pool_needs_refill(struct dynamic_page_pool *pool) +{ + return pool->order && dynamic_pool_count_below_lowmark(pool); +} + +static int system_heap_refill_worker(void *data) +{ + struct dynamic_page_pool **pool_list = data; + int i; + + for (;;) { + for (i = 0; i < NUM_ORDERS; i++) { + if (dynamic_pool_count_below_lowmark(pool_list[i])) + dynamic_page_pool_refill(pool_list[i]); + } + + set_current_state(TASK_INTERRUPTIBLE); + if (unlikely(kthread_should_stop())) { + set_current_state(TASK_RUNNING); + break; + } + schedule(); + + set_current_state(TASK_RUNNING); + } + + return 0; +} + +static int system_heap_create_refill_worker(struct qcom_system_heap *sys_heap, const char *name) +{ + struct task_struct *refill_worker; + struct sched_attr attr = { .sched_nice = DYNAMIC_POOL_KTHREAD_NICE_VAL }; + int ret; + int i; + + refill_worker = kthread_run(system_heap_refill_worker, sys_heap->pool_list, + "%s-pool-refill-thread", name); + if (IS_ERR(refill_worker)) { + pr_err("%s: failed to create %s-pool-refill-thread: %ld\n", + __func__, name, PTR_ERR(refill_worker)); + return PTR_ERR(refill_worker); + } + + ret = sched_setattr(refill_worker, &attr); + if (ret) { + pr_warn("%s: failed to set task priority for %s-pool-refill-thread: ret = %d\n", + __func__, name, ret); + kthread_stop(refill_worker); + return ret; + } + + for (i = 0; i < NUM_ORDERS; i++) + sys_heap->pool_list[i]->refill_worker = refill_worker; + + return ret; +} + +static void system_heap_destroy_refill_worker(struct qcom_system_heap *sys_heap) +{ + kthread_stop(sys_heap->pool_list[0]->refill_worker); +} +#else +static bool dynamic_pool_needs_refill(struct dynamic_page_pool *pool) +{ + return false; +} + +static int system_heap_create_refill_worker(struct qcom_system_heap *sys_heap, const char *name) +{ + return 0; +} + +static void system_heap_destroy_refill_worker(struct qcom_system_heap *sys_heap) +{ +} +#endif + +static int system_heap_clear_pages(struct page **pages, int num, pgprot_t pgprot) +{ + void *addr = vmap(pages, num, VM_MAP, pgprot); + + if (!addr) + return -ENOMEM; + memset(addr, 0, PAGE_SIZE * num); + vunmap(addr); + return 0; +} + +static int system_heap_zero_buffer(struct qcom_sg_buffer *buffer) +{ + struct sg_table *sgt = &buffer->sg_table; + struct sg_page_iter piter; + struct page *pages[32]; + int p = 0; + int ret = 0; + + for_each_sgtable_page(sgt, &piter, 0) { + pages[p++] = sg_page_iter_page(&piter); + if (p == ARRAY_SIZE(pages)) { + ret = system_heap_clear_pages(pages, p, PAGE_KERNEL); + if (ret) + return ret; + p = 0; + } + } + if (p) + ret = system_heap_clear_pages(pages, p, PAGE_KERNEL); + + return ret; +} + +static void system_heap_free(struct qcom_sg_buffer *buffer) +{ + struct qcom_system_heap *sys_heap; + struct sg_table *table; + struct scatterlist *sg; + int i, j; + + sys_heap = dma_heap_get_drvdata(buffer->heap); + table = &buffer->sg_table; + + /* Zero the buffer pages before adding back to the pool */ + system_heap_zero_buffer(buffer); + + for_each_sg(table->sgl, sg, table->nents, i) { + struct page *page = sg_page(sg); + + for (j = 0; j < NUM_ORDERS; j++) { + if (compound_order(page) == orders[j]) + break; + } + dynamic_page_pool_free(sys_heap->pool_list[j], page); + } + sg_free_table(table); + kfree(buffer); +} + +struct page *qcom_sys_heap_alloc_largest_available(struct dynamic_page_pool **pools, + unsigned long size, + unsigned int max_order) +{ + struct page *page = NULL; + int i; + + for (i = 0; i < NUM_ORDERS; i++) { + if (size < (PAGE_SIZE << orders[i])) + continue; + if (max_order < orders[i]) + continue; + + mutex_lock(&pools[i]->mutex); + if (pools[i]->high_count) + page = dynamic_page_pool_remove(pools[i], true); + else if (pools[i]->low_count) + page = dynamic_page_pool_remove(pools[i], false); + mutex_unlock(&pools[i]->mutex); + + if (!page) + page = alloc_pages(pools[i]->gfp_mask, pools[i]->order); + if (!page) + continue; + + if (dynamic_pool_needs_refill(pools[i])) + wake_up_process(pools[i]->refill_worker); + + return page; + } + return NULL; +} + +static struct dma_buf *system_heap_allocate(struct dma_heap *heap, + unsigned long len, + unsigned long fd_flags, + unsigned long heap_flags) +{ + struct qcom_system_heap *sys_heap; + struct qcom_sg_buffer *buffer; + DEFINE_DMA_BUF_EXPORT_INFO(exp_info); + unsigned long size_remaining = len; + unsigned int max_order = orders[0]; + struct dma_buf *dmabuf; + struct sg_table *table; + struct scatterlist *sg; + struct list_head pages; + struct page *page, *tmp_page; + int i, ret = -ENOMEM; + + buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); + if (!buffer) + return ERR_PTR(-ENOMEM); + + sys_heap = dma_heap_get_drvdata(heap); + + INIT_LIST_HEAD(&buffer->attachments); + mutex_init(&buffer->lock); + buffer->heap = heap; + buffer->len = len; + buffer->uncached = sys_heap->uncached; + buffer->free = system_heap_free; + + INIT_LIST_HEAD(&pages); + i = 0; + while (size_remaining > 0) { + /* + * Avoid trying to allocate memory if the process + * has been killed by SIGKILL + */ + if (fatal_signal_pending(current)) + goto free_buffer; + + page = qcom_sys_heap_alloc_largest_available(sys_heap->pool_list, + size_remaining, + max_order); + if (!page) + goto free_buffer; + + list_add_tail(&page->lru, &pages); + size_remaining -= page_size(page); + max_order = compound_order(page); + i++; + } + + table = &buffer->sg_table; + if (sg_alloc_table(table, i, GFP_KERNEL)) + goto free_buffer; + + sg = table->sgl; + list_for_each_entry_safe(page, tmp_page, &pages, lru) { + sg_set_page(sg, page, page_size(page), 0); + sg = sg_next(sg); + list_del(&page->lru); + } + + /* + * For uncached buffers, we need to initially flush cpu cache, since + * the __GFP_ZERO on the allocation means the zeroing was done by the + * cpu and thus it is likely cached. Map (and implicitly flush) and + * unmap it now so we don't get corruption later on. + */ + if (buffer->uncached) { + dma_map_sgtable(dma_heap_get_dev(heap), table, DMA_BIDIRECTIONAL, 0); + dma_unmap_sgtable(dma_heap_get_dev(heap), table, DMA_BIDIRECTIONAL, 0); + } + + buffer->vmperm = mem_buf_vmperm_alloc(table); + + if (IS_ERR(buffer->vmperm)) { + ret = PTR_ERR(buffer->vmperm); + goto free_sg; + } + + /* create the dmabuf */ + exp_info.exp_name = dma_heap_get_name(heap); + exp_info.size = buffer->len; + exp_info.flags = fd_flags; + exp_info.priv = buffer; + dmabuf = mem_buf_dma_buf_export(&exp_info, &qcom_sg_buf_ops); + if (IS_ERR(dmabuf)) { + ret = PTR_ERR(dmabuf); + goto vmperm_release; + } + + return dmabuf; + +vmperm_release: + mem_buf_vmperm_release(buffer->vmperm); +free_sg: + sg_free_table(table); +free_buffer: + list_for_each_entry_safe(page, tmp_page, &pages, lru) + __free_pages(page, compound_order(page)); + kfree(buffer); + + return ERR_PTR(ret); +} + +static const struct dma_heap_ops system_heap_ops = { + .allocate = system_heap_allocate, +}; + +void qcom_system_heap_create(const char *name, const char *system_alias, bool uncached) +{ + struct dma_heap_export_info exp_info; + struct dma_heap *heap; + struct qcom_system_heap *sys_heap; + int ret; + + ret = dynamic_page_pool_init_shrinker(); + if (ret) + goto out; + + sys_heap = kzalloc(sizeof(*sys_heap), GFP_KERNEL); + if (!sys_heap) { + ret = -ENOMEM; + goto out; + } + + exp_info.name = name; + exp_info.ops = &system_heap_ops; + exp_info.priv = sys_heap; + + sys_heap->uncached = uncached; + + sys_heap->pool_list = dynamic_page_pool_create_pools(0, NULL); + if (IS_ERR(sys_heap->pool_list)) { + ret = PTR_ERR(sys_heap->pool_list); + goto free_heap; + } + + ret = system_heap_create_refill_worker(sys_heap, name); + if (ret) + goto free_pools; + + heap = dma_heap_add(&exp_info); + if (IS_ERR(heap)) { + ret = PTR_ERR(heap); + goto stop_worker; + } + + if (uncached) + dma_coerce_mask_and_coherent(dma_heap_get_dev(heap), + DMA_BIT_MASK(64)); + + pr_info("%s: DMA-BUF Heap: Created '%s'\n", __func__, name); + + if (system_alias != NULL) { + exp_info.name = system_alias; + + heap = dma_heap_add(&exp_info); + if (IS_ERR(heap)) { + pr_err("%s: Failed to create '%s', error is %d\n", __func__, + system_alias, PTR_ERR(heap)); + return; + } + + dma_coerce_mask_and_coherent(dma_heap_get_dev(heap), DMA_BIT_MASK(64)); + + pr_info("%s: DMA-BUF Heap: Created '%s'\n", __func__, system_alias); + } + + return; + +stop_worker: + system_heap_destroy_refill_worker(sys_heap); + +free_pools: + dynamic_page_pool_release_pools(sys_heap->pool_list); + +free_heap: + kfree(sys_heap); + +out: + pr_err("%s: Failed to create '%s', error is %d\n", __func__, name, ret); +} diff --git a/drivers/dma-buf/heaps/qcom_system_heap.h b/drivers/dma-buf/heaps/qcom_system_heap.h new file mode 100644 index 000000000000..01f9dad91474 --- /dev/null +++ b/drivers/dma-buf/heaps/qcom_system_heap.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#ifndef _QCOM_SYSTEM_HEAP_H +#define _QCOM_SYSTEM_HEAP_H + +#include +#include +#include "qcom_dynamic_page_pool.h" + +struct qcom_system_heap { + int uncached; + struct dynamic_page_pool **pool_list; +}; + +#ifdef CONFIG_QCOM_DMABUF_HEAPS_SYSTEM +void qcom_system_heap_create(const char *name, const char *system_alias, bool uncached); +struct page *qcom_sys_heap_alloc_largest_available(struct dynamic_page_pool **pools, + unsigned long size, + unsigned int max_order); +#else +struct page *qcom_sys_heap_alloc_largest_available(struct dynamic_page_pool **pools, + unsigned long size, + unsigned int max_order) +{ + return ERR_PTR(-EOPNOTSUPP); +} + +static void qcom_system_heap_create(const char *name, const char *system_alias, bool uncached) +{ + +} +#endif + +#endif /* _QCOM_SYSTEM_HEAP_H */ diff --git a/drivers/iommu/msm_dma_iommu_mapping.c b/drivers/iommu/msm_dma_iommu_mapping.c new file mode 100644 index 000000000000..d06dd1507345 --- /dev/null +++ b/drivers/iommu/msm_dma_iommu_mapping.c @@ -0,0 +1,480 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2015-2019, 2021, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +/** + * struct msm_iommu_map - represents a mapping of an ion buffer to an iommu + * @lnode - list node to exist in the buffer's list of iommu mappings + * @dev - Device this is mapped to. Used as key + * @sgl - The scatterlist for this mapping + * @nents - Number of entries in sgl + * @dir - The direction for the map. + * @meta - Backpointer to the meta this guy belongs to. + * @ref - for reference counting this mapping + * @attrs - dma mapping attributes + * @buf_start_addr - address of start of buffer + * + * Represents a mapping of one dma_buf buffer to a particular device + * and address range. There may exist other mappings of this buffer in + * different devices. All mappings will have the same cacheability and security. + */ +struct msm_iommu_map { + struct list_head lnode; + struct rb_node node; + struct device *dev; + struct scatterlist *sgl; + unsigned int nents; + enum dma_data_direction dir; + struct msm_iommu_meta *meta; + struct kref ref; + unsigned long attrs; + dma_addr_t buf_start_addr; +}; + +struct msm_iommu_meta { + struct rb_node node; + struct list_head iommu_maps; + struct kref ref; + struct mutex lock; + void *buffer; +}; + +static struct rb_root iommu_root; +static DEFINE_MUTEX(msm_iommu_map_mutex); + +static void msm_iommu_meta_add(struct msm_iommu_meta *meta) +{ + struct rb_root *root = &iommu_root; + struct rb_node **p = &root->rb_node; + struct rb_node *parent = NULL; + struct msm_iommu_meta *entry; + + while (*p) { + parent = *p; + entry = rb_entry(parent, struct msm_iommu_meta, node); + + if (meta->buffer < entry->buffer) + p = &(*p)->rb_left; + else if (meta->buffer > entry->buffer) + p = &(*p)->rb_right; + else + pr_err("%s: dma_buf %pK already exists\n", __func__, + entry->buffer); + } + + rb_link_node(&meta->node, parent, p); + rb_insert_color(&meta->node, root); +} + +static struct msm_iommu_meta *msm_iommu_meta_lookup(void *buffer) +{ + struct rb_root *root = &iommu_root; + struct rb_node **p = &root->rb_node; + struct rb_node *parent = NULL; + struct msm_iommu_meta *entry = NULL; + + while (*p) { + parent = *p; + entry = rb_entry(parent, struct msm_iommu_meta, node); + + if (buffer < entry->buffer) + p = &(*p)->rb_left; + else if (buffer > entry->buffer) + p = &(*p)->rb_right; + else + return entry; + } + + return NULL; +} + +static void msm_iommu_add(struct msm_iommu_meta *meta, + struct msm_iommu_map *iommu) +{ + INIT_LIST_HEAD(&iommu->lnode); + list_add(&iommu->lnode, &meta->iommu_maps); +} + + +static struct msm_iommu_map *msm_iommu_lookup(struct msm_iommu_meta *meta, + struct device *dev) +{ + struct msm_iommu_map *entry; + + list_for_each_entry(entry, &meta->iommu_maps, lnode) { + if (entry->dev == dev) + return entry; + } + + return NULL; +} + +static struct msm_iommu_meta *msm_iommu_meta_create(struct dma_buf *dma_buf) +{ + struct msm_iommu_meta *meta; + + meta = kzalloc(sizeof(*meta), GFP_KERNEL); + + if (!meta) + return ERR_PTR(-ENOMEM); + + INIT_LIST_HEAD(&meta->iommu_maps); + meta->buffer = dma_buf->priv; + kref_init(&meta->ref); + mutex_init(&meta->lock); + msm_iommu_meta_add(meta); + + return meta; +} + +static void msm_iommu_meta_put(struct msm_iommu_meta *meta); + +static struct scatterlist *clone_sgl(struct scatterlist *sg, int nents) +{ + struct scatterlist *next, *s; + int i; + struct sg_table table; + + if (sg_alloc_table(&table, nents, GFP_KERNEL)) + return NULL; + next = table.sgl; + for_each_sg(sg, s, nents, i) { + *next = *s; + next = sg_next(next); + } + return table.sgl; +} + +static inline int __msm_dma_map_sg(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir, + struct dma_buf *dma_buf, + unsigned long attrs) +{ + struct msm_iommu_map *iommu_map; + struct msm_iommu_meta *iommu_meta = NULL; + int ret = 0; + bool extra_meta_ref_taken = false; + int late_unmap = !(attrs & DMA_ATTR_NO_DELAYED_UNMAP); + + mutex_lock(&msm_iommu_map_mutex); + iommu_meta = msm_iommu_meta_lookup(dma_buf->priv); + + if (!iommu_meta) { + iommu_meta = msm_iommu_meta_create(dma_buf); + + if (IS_ERR(iommu_meta)) { + mutex_unlock(&msm_iommu_map_mutex); + ret = PTR_ERR(iommu_meta); + goto out; + } + if (late_unmap) { + kref_get(&iommu_meta->ref); + extra_meta_ref_taken = true; + } + } else { + kref_get(&iommu_meta->ref); + } + + mutex_unlock(&msm_iommu_map_mutex); + + mutex_lock(&iommu_meta->lock); + iommu_map = msm_iommu_lookup(iommu_meta, dev); + if (!iommu_map) { + iommu_map = kmalloc(sizeof(*iommu_map), GFP_KERNEL); + + if (!iommu_map) { + ret = -ENOMEM; + goto out_unlock; + } + + ret = dma_map_sg_attrs(dev, sg, nents, dir, attrs); + if (!ret) { + kfree(iommu_map); + goto out_unlock; + } + + iommu_map->sgl = clone_sgl(sg, nents); + if (!iommu_map->sgl) { + kfree(iommu_map); + ret = -ENOMEM; + goto out_unlock; + } + iommu_map->nents = nents; + iommu_map->dev = dev; + iommu_map->dir = dir; + iommu_map->attrs = attrs; + iommu_map->buf_start_addr = sg_phys(sg); + + kref_init(&iommu_map->ref); + if (late_unmap) + kref_get(&iommu_map->ref); + iommu_map->meta = iommu_meta; + msm_iommu_add(iommu_meta, iommu_map); + + } else { + if (nents == iommu_map->nents && + dir == iommu_map->dir && + (attrs & ~DMA_ATTR_SKIP_CPU_SYNC) == + (iommu_map->attrs & ~DMA_ATTR_SKIP_CPU_SYNC) && + sg_phys(sg) == iommu_map->buf_start_addr) { + struct scatterlist *sg_tmp = sg; + struct scatterlist *map_sg; + int i; + + for_each_sg(iommu_map->sgl, map_sg, nents, i) { + sg_dma_address(sg_tmp) = sg_dma_address(map_sg); + sg_dma_len(sg_tmp) = sg_dma_len(map_sg); + if (sg_dma_len(map_sg) == 0) + break; + + sg_tmp = sg_next(sg_tmp); + if (sg_tmp == NULL) + break; + } + + kref_get(&iommu_map->ref); + + if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) + dma_sync_sg_for_device(dev, iommu_map->sgl, + iommu_map->nents, iommu_map->dir); + + if (dev_is_dma_coherent(dev)) + /* + * Ensure all outstanding changes for coherent + * buffers are applied to the cache before any + * DMA occurs. + */ + dmb(ish); + ret = nents; + } else { + bool start_diff = (sg_phys(sg) != + iommu_map->buf_start_addr); + + dev_err(dev, "lazy map request differs:\n" + "req dir:%d, original dir:%d\n" + "req nents:%d, original nents:%d\n" + "req map attrs:%lu, original map attrs:%lu\n" + "req buffer start address differs:%d\n", + dir, iommu_map->dir, nents, + iommu_map->nents, attrs, iommu_map->attrs, + start_diff); + ret = -EINVAL; + } + } + mutex_unlock(&iommu_meta->lock); + return ret; + +out_unlock: + mutex_unlock(&iommu_meta->lock); +out: + if (!IS_ERR(iommu_meta)) { + if (extra_meta_ref_taken) + msm_iommu_meta_put(iommu_meta); + msm_iommu_meta_put(iommu_meta); + } + return ret; + +} + +/* + * We are not taking a reference to the dma_buf here. It is expected that + * clients hold reference to the dma_buf until they are done with mapping and + * unmapping. + */ +int msm_dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents, + enum dma_data_direction dir, struct dma_buf *dma_buf, + unsigned long attrs) +{ + int ret; + + if (IS_ERR_OR_NULL(dev)) { + pr_err("%s: dev pointer is invalid\n", __func__); + return -EINVAL; + } + + if (IS_ERR_OR_NULL(sg)) { + pr_err("%s: sg table pointer is invalid\n", __func__); + return -EINVAL; + } + + if (IS_ERR_OR_NULL(dma_buf)) { + pr_err("%s: dma_buf pointer is invalid\n", __func__); + return -EINVAL; + } + + ret = __msm_dma_map_sg(dev, sg, nents, dir, dma_buf, attrs); + + return ret; +} +EXPORT_SYMBOL(msm_dma_map_sg_attrs); + +static void msm_iommu_meta_destroy(struct kref *kref) +{ + struct msm_iommu_meta *meta = container_of(kref, struct msm_iommu_meta, + ref); + + if (!list_empty(&meta->iommu_maps)) { + WARN(1, "%s: DMA Buffer %pK being destroyed with outstanding iommu mappings!\n", + __func__, meta->buffer); + } + rb_erase(&meta->node, &iommu_root); + kfree(meta); +} + +static void msm_iommu_meta_put(struct msm_iommu_meta *meta) +{ + /* + * Need to lock here to prevent race against map/unmap + */ + mutex_lock(&msm_iommu_map_mutex); + kref_put(&meta->ref, msm_iommu_meta_destroy); + mutex_unlock(&msm_iommu_map_mutex); +} + +static void msm_iommu_map_release(struct kref *kref) +{ + struct msm_iommu_map *map = container_of(kref, struct msm_iommu_map, + ref); + struct sg_table table; + + table.nents = table.orig_nents = map->nents; + table.sgl = map->sgl; + list_del(&map->lnode); + + /* Skip an additional cache maintenance on the dma unmap path */ + if (!(map->attrs & DMA_ATTR_SKIP_CPU_SYNC)) + map->attrs |= DMA_ATTR_SKIP_CPU_SYNC; + dma_unmap_sg_attrs(map->dev, map->sgl, map->nents, map->dir, + map->attrs); + sg_free_table(&table); + kfree(map); +} + +void msm_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl, + int nents, enum dma_data_direction dir, + struct dma_buf *dma_buf, unsigned long attrs) +{ + struct msm_iommu_map *iommu_map; + struct msm_iommu_meta *meta; + + mutex_lock(&msm_iommu_map_mutex); + meta = msm_iommu_meta_lookup(dma_buf->priv); + if (!meta) { + WARN(1, "%s: (%pK) was never mapped\n", __func__, dma_buf); + mutex_unlock(&msm_iommu_map_mutex); + goto out; + + } + mutex_unlock(&msm_iommu_map_mutex); + + mutex_lock(&meta->lock); + iommu_map = msm_iommu_lookup(meta, dev); + + if (!iommu_map) { + WARN(1, "%s: (%pK) was never mapped for device %p\n", __func__, + dma_buf, dev); + mutex_unlock(&meta->lock); + goto out; + } + + if (dir != iommu_map->dir) + WARN(1, "%s: (%pK) dir:%d differs from original dir:%d\n", + __func__, dma_buf, dir, iommu_map->dir); + + if (attrs && ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)) + dma_sync_sg_for_cpu(dev, iommu_map->sgl, iommu_map->nents, dir); + + iommu_map->attrs = attrs; + kref_put(&iommu_map->ref, msm_iommu_map_release); + mutex_unlock(&meta->lock); + + msm_iommu_meta_put(meta); + +out: + return; +} +EXPORT_SYMBOL(msm_dma_unmap_sg_attrs); + +int msm_dma_unmap_all_for_dev(struct device *dev) +{ + int ret = 0; + struct msm_iommu_meta *meta; + struct rb_root *root; + struct rb_node *meta_node; + + mutex_lock(&msm_iommu_map_mutex); + root = &iommu_root; + meta_node = rb_first(root); + while (meta_node) { + struct msm_iommu_map *iommu_map; + struct msm_iommu_map *iommu_map_next; + + meta = rb_entry(meta_node, struct msm_iommu_meta, node); + mutex_lock(&meta->lock); + list_for_each_entry_safe(iommu_map, iommu_map_next, + &meta->iommu_maps, lnode) + if (iommu_map->dev == dev) + if (!kref_put(&iommu_map->ref, + msm_iommu_map_release)) + ret = -EINVAL; + + mutex_unlock(&meta->lock); + meta_node = rb_next(meta_node); + } + mutex_unlock(&msm_iommu_map_mutex); + + return ret; +} +EXPORT_SYMBOL(msm_dma_unmap_all_for_dev); + +/* + * Only to be called by ION code when a buffer is freed + */ +void msm_dma_buf_freed(void *buffer) +{ + struct msm_iommu_map *iommu_map; + struct msm_iommu_map *iommu_map_next; + struct msm_iommu_meta *meta; + + mutex_lock(&msm_iommu_map_mutex); + meta = msm_iommu_meta_lookup(buffer); + if (!meta) { + /* Already unmapped (assuming no late unmapping) */ + mutex_unlock(&msm_iommu_map_mutex); + return; + } + mutex_unlock(&msm_iommu_map_mutex); + + mutex_lock(&meta->lock); + + list_for_each_entry_safe(iommu_map, iommu_map_next, &meta->iommu_maps, + lnode) + kref_put(&iommu_map->ref, msm_iommu_map_release); + + if (!list_empty(&meta->iommu_maps)) { + WARN(1, "%s: DMA buffer %pK destroyed with outstanding iommu mappings\n", + __func__, meta->buffer); + } + + INIT_LIST_HEAD(&meta->iommu_maps); + mutex_unlock(&meta->lock); + + msm_iommu_meta_put(meta); +} +EXPORT_SYMBOL(msm_dma_buf_freed); + +MODULE_LICENSE("GPL v2"); diff --git a/include/linux/msm_dma_iommu_mapping.h b/include/linux/msm_dma_iommu_mapping.h new file mode 100644 index 000000000000..553bf7fabe22 --- /dev/null +++ b/include/linux/msm_dma_iommu_mapping.h @@ -0,0 +1,144 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2015-2016, 2018, 2021 The Linux Foundation. All rights reserved. + */ + +#ifndef _LINUX_MSM_DMA_IOMMU_MAPPING_H +#define _LINUX_MSM_DMA_IOMMU_MAPPING_H + +#include +#include +#include +#include +#include + +#if IS_ENABLED(CONFIG_QCOM_LAZY_MAPPING) +/* + * This function is not taking a reference to the dma_buf here. It is expected + * that clients hold reference to the dma_buf until they are done with mapping + * and unmapping. + */ +int msm_dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents, + enum dma_data_direction dir, struct dma_buf *dma_buf, + unsigned long attrs); + +/* + * This function takes an extra reference to the dma_buf. + * What this means is that calling msm_dma_unmap_sg will not result in buffer's + * iommu mapping being removed, which means that subsequent calls to lazy map + * will simply re-use the existing iommu mapping. + * The iommu unmapping of the buffer will occur when the ION buffer is + * destroyed. + * Using lazy mapping can provide a performance benefit because subsequent + * mappings are faster. + * + * The limitation of using this API are that all subsequent iommu mappings + * must be the same as the original mapping, ie they must map the same part of + * the buffer with the same dma data direction. Also there can't be multiple + * mappings of different parts of the buffer. + */ +static inline int msm_dma_map_sg_lazy(struct device *dev, + struct scatterlist *sg, int nents, + enum dma_data_direction dir, + struct dma_buf *dma_buf) +{ + return msm_dma_map_sg_attrs(dev, sg, nents, dir, dma_buf, 0); +} + +static inline int msm_dma_map_sg(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir, + struct dma_buf *dma_buf) +{ + unsigned long attrs; + + attrs = DMA_ATTR_NO_DELAYED_UNMAP; + return msm_dma_map_sg_attrs(dev, sg, nents, dir, dma_buf, attrs); +} + +void msm_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl, + int nents, enum dma_data_direction dir, + struct dma_buf *dma_buf, unsigned long attrs); + +static inline int msm_dma_map_sgtable(struct device *dev, struct sg_table *sgt, + enum dma_data_direction dir, + struct dma_buf *dma_buf, unsigned long attrs) +{ + int nents; + + nents = msm_dma_map_sg_attrs(dev, sgt->sgl, sgt->orig_nents, dir, dma_buf, attrs); + if (nents <= 0) + return -EINVAL; + + sgt->nents = nents; + return 0; +} + +static inline void msm_dma_unmap_sgtable(struct device *dev, struct sg_table *sgt, + enum dma_data_direction dir, + struct dma_buf *dma_buf, unsigned long attrs) +{ + msm_dma_unmap_sg_attrs(dev, sgt->sgl, sgt->nents, dir, dma_buf, attrs); +} + +int msm_dma_unmap_all_for_dev(struct device *dev); + +/* + * Below is private function only to be called by framework (ION) and not by + * clients. + */ +void msm_dma_buf_freed(void *buffer); + +#else /*CONFIG_QCOM_LAZY_MAPPING*/ + +static inline int msm_dma_map_sg_attrs(struct device *dev, + struct scatterlist *sg, int nents, + enum dma_data_direction dir, struct dma_buf *dma_buf, + unsigned long attrs) +{ + return -EINVAL; +} + +static inline void +msm_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl, + int nents, enum dma_data_direction dir, + struct dma_buf *dma_buf, unsigned long attrs) +{ +} + +static inline int msm_dma_map_sg_lazy(struct device *dev, + struct scatterlist *sg, int nents, + enum dma_data_direction dir, + struct dma_buf *dma_buf) +{ + return -EINVAL; +} + +static inline int msm_dma_map_sg(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir, + struct dma_buf *dma_buf) +{ + return -EINVAL; +} + +static inline int msm_dma_map_sgtable(struct device *dev, struct sg_table *sgt, + enum dma_data_direction dir, + struct dma_buf *dma_buf, unsigned long attrs) +{ + return -EINVAL; +} + +static inline void msm_dma_unmap_sgtable(struct device *dev, struct sg_table *sgt, + enum dma_data_direction dir, + struct dma_buf *dma_buf, unsigned long attrs) +{ +} + +static inline int msm_dma_unmap_all_for_dev(struct device *dev) +{ + return 0; +} + +static inline void msm_dma_buf_freed(void *buffer) {} +#endif /*CONFIG_QCOM_LAZY_MAPPING*/ + +#endif diff --git a/include/linux/qcom_dma_heap.h b/include/linux/qcom_dma_heap.h new file mode 100644 index 000000000000..177ed7ef9f75 --- /dev/null +++ b/include/linux/qcom_dma_heap.h @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ +#ifndef _QCOM_DMA_HEAP_H +#define _QCOM_DMA_HEAP_H + +#include +#include +#include + +#include +#include + +/* Heap flags */ +#define QCOM_DMA_HEAP_FLAG_CACHED BIT(1) + +#define QCOM_DMA_HEAP_FLAG_CP_TRUSTED_VM BIT(15) +#define QCOM_DMA_HEAP_FLAG_CP_TZ BIT(16) +#define QCOM_DMA_HEAP_FLAG_CP_TOUCH BIT(17) +#define QCOM_DMA_HEAP_FLAG_CP_BITSTREAM BIT(18) +#define QCOM_DMA_HEAP_FLAG_CP_PIXEL BIT(19) +#define QCOM_DMA_HEAP_FLAG_CP_NON_PIXEL BIT(20) +#define QCOM_DMA_HEAP_FLAG_CP_CAMERA BIT(21) +#define QCOM_DMA_HEAP_FLAG_CP_HLOS BIT(22) +#define QCOM_DMA_HEAP_FLAG_CP_SPSS_SP BIT(23) +#define QCOM_DMA_HEAP_FLAG_CP_SPSS_SP_SHARED BIT(24) +#define QCOM_DMA_HEAP_FLAG_CP_SEC_DISPLAY BIT(25) +#define QCOM_DMA_HEAP_FLAG_CP_APP BIT(26) +#define QCOM_DMA_HEAP_FLAG_CP_CAMERA_PREVIEW BIT(27) +#define QCOM_DMA_HEAP_FLAG_CP_MSS_MSA BIT(28) +#define QCOM_DMA_HEAP_FLAG_CP_CDSP BIT(29) +#define QCOM_DMA_HEAP_FLAG_CP_SPSS_HLOS_SHARED BIT(30) + +#define QCOM_DMA_HEAP_FLAGS_CP_MASK GENMASK(30, 15) + +#define QCOM_DMA_HEAP_FLAG_SECURE BIT(31) + +/** + * carveout_heap_add_memory - Dynamically add reserved heap memory + * @heap_name: The name of the DMA-BUF Heap we're giving memory to + * @sgt: An SGT corresponding to the memory adding to the heap + * @cookie: Cookie for managing refcount. + * @get: Function callback for increasing refcount. + * @put: Function callback for decreasing refcount. + * + * Return: 0 on success, a negative error value otherwise. + */ +int carveout_heap_add_memory(char *heap_name, struct sg_table *sgt, void *cookie, + int (*get)(void *), void (*put)(void *)); + + +/** + * carveout_heap_remove_memory - Dynamically remove reserved heap memory + * @heap_name: The name of the DMA-BUF Heap we're taking memory away from + * @sgt: An SGT corresponding to the memory we're removing from the heap + * + * Return: 0 on success, a negative error value otherwise. + */ +int carveout_heap_remove_memory(char *heap_name, struct sg_table *sgt); + +struct dma_buf_heap_prefetch_region { + u64 size; + struct dma_heap *heap; +}; + +int qcom_secure_system_heap_prefetch(struct dma_buf_heap_prefetch_region *regions, + size_t nr_regions); + +int qcom_secure_system_heap_drain(struct dma_buf_heap_prefetch_region *regions, + size_t nr_regions); + +/** + * dma_buf_heap_hyp_assign - wrapper function for hyp-assigning a dma_buf + * @buf: dma_buf to hyp-assign away from HLOS + * @dest_vmids: array of QCOM_DMA_HEAP_FLAG VMIDs (as defined above) + * @dest_perms: array of PERM_READ/PERM_WRITE/PERM_EXEC permission bits (as + * defined in include/soc/qcom/secure_buffer.h), such that + * dest_perms[i] specifies the permissions for VMID dest_vmids[i] + * @dest_nelems: number of elements in dest_vmids and dest_perms + * + * Return: Temporarily return -EINVAL whilst the functon isn't present. Otherwise, return + * 0 on success, or a negative value returned by hyp_assign_table() on failure. + */ +static inline int dma_buf_heap_hyp_assign(struct dma_buf *buf, int *dest_vmids, int *dest_perms, + int dest_nelems) +{ + return -EINVAL; +} + +/** + * dma_buf_heap_hyp_unassign - wrapper function that hyp-assigns a dma_buf back to HLOS + * @buf: dma_buf to hyp-assign back to HLOS + * + * This function takes a dma_buf, and re-assigns it to HLOS with RWX permissions (at the + * S2 level). The end-points to which the buffer was assigned to are tracked by the flags + * kept in the msm_heap_helper_buffer->flags field (the helper_buffer is accesed through + * dma_buf->priv), so the corresponding VMIDs don't need to be supplied as arguments. + * + * Return: Temporarily return -EINVAL whilst the functon isn't present. Otherwise, return + * 0 on success, or a negative value returned by hyp_assign_table() on failure. + */ +static inline int dma_buf_heap_hyp_unassign(struct dma_buf *buf) +{ + return -EINVAL; +} + +#endif /* _QCOM_DMA_HEAP_H */ diff --git a/include/linux/qcom_dma_heap_dt_constants.h b/include/linux/qcom_dma_heap_dt_constants.h new file mode 100644 index 000000000000..7bfd56fa42e7 --- /dev/null +++ b/include/linux/qcom_dma_heap_dt_constants.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ +#ifndef _QCOM_DMA_HEAP_DT_CONSTATS_H +#define _QCOM_DMA_HEAP_DT_CONSTATS_H + +#define HEAP_TYPE_SECURE_CARVEOUT 0 +#define HEAP_TYPE_CARVEOUT 1 +#define HEAP_TYPE_CMA 2 + +#endif /* _QCOM_DMA_HEAP_DT_CONSTATS_H */ diff --git a/include/uapi/linux/msm_ion.h b/include/uapi/linux/msm_ion.h new file mode 100644 index 000000000000..048707fc26e1 --- /dev/null +++ b/include/uapi/linux/msm_ion.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved. + */ +#ifndef _UAPI_LINUX_MSM_ION_H +#define _UAPI_LINUX_MSM_ION_H + +#include +#include + +/** + * TARGET_ION_ABI_VERSION can be used by user space clients to ensure that at + * compile time only their code which uses the appropriate ION APIs for + * this kernel is included. + */ +#define TARGET_ION_ABI_VERSION 3 + +enum msm_ion_heap_types { + ION_HEAP_TYPE_MSM_START = 16, + ION_HEAP_TYPE_SECURE_DMA = ION_HEAP_TYPE_MSM_START, + ION_HEAP_TYPE_SYSTEM_SECURE, + ION_HEAP_TYPE_HYP_CMA, + ION_HEAP_TYPE_MSM_CARVEOUT, + ION_HEAP_TYPE_SECURE_CARVEOUT, + ION_HEAP_TYPE_MSM_SYSTEM, +}; + +/** + * Flags to be used when allocating from the secure heap for + * content protection + */ +#define ION_FLAG_CP_TRUSTED_VM ION_BIT(15) +/* ION_FLAG_POOL_FORCE_ALLOC uses ION_BIT(16) */ +#define ION_FLAG_CP_TOUCH ION_BIT(17) +#define ION_FLAG_CP_BITSTREAM ION_BIT(18) +#define ION_FLAG_CP_PIXEL ION_BIT(19) +#define ION_FLAG_CP_NON_PIXEL ION_BIT(20) +#define ION_FLAG_CP_CAMERA ION_BIT(21) +#define ION_FLAG_CP_HLOS ION_BIT(22) +#define ION_FLAG_CP_SPSS_SP ION_BIT(23) +#define ION_FLAG_CP_SPSS_SP_SHARED ION_BIT(24) +#define ION_FLAG_CP_SEC_DISPLAY ION_BIT(25) +#define ION_FLAG_CP_APP ION_BIT(26) +#define ION_FLAG_CP_CAMERA_PREVIEW ION_BIT(27) +/* ION_FLAG_ALLOW_NON_CONTIG uses ION_BIT(28) */ +#define ION_FLAG_CP_CDSP ION_BIT(29) +#define ION_FLAG_CP_SPSS_HLOS_SHARED ION_BIT(30) + +#define ION_FLAGS_CP_MASK 0x6FFE8000 + +/** + * Flag to allow non continguous allocation of memory from secure + * heap + */ +#define ION_FLAG_ALLOW_NON_CONTIG ION_BIT(28) + +/** + * Flag to use when allocating to indicate that a heap is secure. + * Do NOT use BIT macro since it is defined in #ifdef __KERNEL__ + */ +#define ION_FLAG_SECURE ION_BIT(31) + +/* + * Used in conjunction with heap which pool memory to force an allocation + * to come from the page allocator directly instead of from the pool allocation + */ +#define ION_FLAG_POOL_FORCE_ALLOC ION_BIT(16) + +/** + * Macro should be used with ion_heap_ids defined above. + */ +#define ION_HEAP(bit) bit + +#define ION_IOC_MSM_MAGIC 'M' + +struct ion_prefetch_regions { + __u64 sizes; + __u32 vmid; + __u32 nr_sizes; +}; + +struct ion_prefetch_data { + __u64 len; + __u64 regions; + __u32 heap_id; + __u32 nr_regions; +}; + +#define ION_IOC_PREFETCH _IOWR(ION_IOC_MSM_MAGIC, 3, \ + struct ion_prefetch_data) + +#define ION_IOC_DRAIN _IOWR(ION_IOC_MSM_MAGIC, 4, \ + struct ion_prefetch_data) + +#endif /* _UAPI_LINUX_MSM_ION_H */ diff --git a/include/uapi/linux/msm_ion_ids.h b/include/uapi/linux/msm_ion_ids.h new file mode 100644 index 000000000000..a6d6233e9db9 --- /dev/null +++ b/include/uapi/linux/msm_ion_ids.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0-only WITH Linux-syscall-note */ +/* + * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. + */ +#ifndef _MSM_ION_IDS_H +#define _MSM_ION_IDS_H + +#define ION_BIT(nr) (1U << (nr)) + +/** + * These are the only ids that should be used for Ion heap ids. + * The ids listed are the order in which allocation will be attempted + * if specified. Don't swap the order of heap ids unless you know what + * you are doing! + * Id's are spaced by purpose to allow new Id's to be inserted in-between (for + * possible fallbacks) + */ + +/* ION_BIT(0) is reserved for the generic system heap. */ +#define ION_QSECOM_TA_HEAP_ID ION_BIT(1) +#define ION_CAMERA_HEAP_ID ION_BIT(2) +#define ION_DISPLAY_HEAP_ID ION_BIT(3) +#define ION_ADSP_HEAP_ID ION_BIT(4) +#define ION_AUDIO_ML_HEAP_ID ION_BIT(5) +#define ION_USER_CONTIG_HEAP_ID ION_BIT(6) +#define ION_QSECOM_HEAP_ID ION_BIT(7) +#define ION_AUDIO_HEAP_ID ION_BIT(8) +#define ION_CP_MM_HEAP_ID ION_BIT(9) +#define ION_SECURE_HEAP_ID ION_BIT(10) +#define ION_SECURE_DISPLAY_HEAP_ID ION_BIT(11) +#define ION_SPSS_HEAP_ID ION_BIT(14) +#define ION_SECURE_CARVEOUT_HEAP_ID ION_BIT(15) +#define ION_TUI_CARVEOUT_HEAP_ID ION_BIT(16) +#define ION_SYSTEM_HEAP_ID ION_BIT(25) +#define ION_HEAP_ID_RESERVED ION_BIT(31) + +#endif /* _MSM_ION_IDS_H */