Merge "dma-heap: qcom: Take a snapshot of our DMA-BUF heap driver from msm-5.15"

drivers/dma-buf/heaps/Kconfig

@@ -12,3 +12,66 @@ config DMABUF_HEAPS_CMA
 	  Choose this option to enable dma-buf CMA heap. This heap is backed
 	  by the Contiguous Memory Allocator (CMA). If your system has these
 	  regions, you should say Y here.
+
+config QCOM_DMABUF_HEAPS
+	tristate "QCOM DMA-BUF Heaps"
+	depends on DMABUF_HEAPS
+	help
+	  Choose this option to build the dma-buf heap module. Other
+	  heaps can subsequently be compiled into this module by enabling
+	  the appropriate defconfig option.
+	  If in doubt, say M here.
+
+config QCOM_DMABUF_HEAPS_SYSTEM
+	bool "QCOM DMA-BUF System Heap"
+	depends on QCOM_DMABUF_HEAPS
+	help
+	  Choose this option to build the QCOM DMA-BUF system
+	  heap. This heap combines the built-in system heap
+	  along with other optimizatons. If in doubt, say Y
+	  here.
+
+config QCOM_DMABUF_HEAPS_PAGE_POOL_REFILL
+	bool "QCOM DMA-BUF Page Pool Refill"
+	depends on QCOM_DMABUF_HEAPS_SYSTEM
+	help
+	  Choose this option to enable the page pool auto refill
+	  feature for the system heap's page pools. When enabled,
+	  a thread is created which wakes up to refill the page pools
+	  when they fall below a particular threshold, which in turn
+	  results in smaller allocation latencies. If in doubt, say Y
+	  here.
+
+config QCOM_DMABUF_HEAPS_SYSTEM_SECURE
+	bool "QCOM DMA-BUF System Secure Heap"
+	depends on QCOM_DMABUF_HEAPS && QCOM_SECURE_BUFFER
+	depends on QCOM_DMABUF_HEAPS_SYSTEM
+	help
+	  Choose this option to build the QCOM DMA-BUF system
+	  secure heap. This heap, which combines the built-in
+	  system heap along with other optimizations, assigns
+	  memory that is secure by default. If in doubt, say Y
+	  here.
+
+config QCOM_DMABUF_HEAPS_SYSTEM_UNCACHED
+	bool "QCOM DMA-BUF Uncached System Heap"
+	depends on QCOM_DMABUF_HEAPS && QCOM_DMABUF_HEAPS_SYSTEM
+	help
+	  Choose this option to create an uncached QCOM DMA-BUF
+	  system heap. This heap is equivalent to our system heap,
+	  such that the memory is mapped as uncached. If in doubt,
+	  say Y here.
+
+config QCOM_DMABUF_HEAPS_CMA
+	bool "QCOM DMA-BUF CMA Heap"
+	depends on QCOM_DMABUF_HEAPS && DMA_CMA
+	help
+	  Choose this option to build the QCOM DMA-BUF CMA heap.
+	  If in doubt, say Y here.
+
+config QCOM_DMABUF_HEAPS_CARVEOUT
+	bool "QCOM dma-buf Carveout Heap"
+	depends on QCOM_DMABUF_HEAPS
+	help
+	  Choose this option to build the QCOM DMA-BUF Carveout heap.
+	  If in doubt, say Y here.

drivers/dma-buf/heaps/Makefile

@@ -1,3 +1,14 @@
 # SPDX-License-Identifier: GPL-2.0
 obj-$(CONFIG_DMABUF_HEAPS_SYSTEM)	+= system_heap.o
 obj-$(CONFIG_DMABUF_HEAPS_CMA)		+= cma_heap.o
+obj-$(CONFIG_QCOM_DMABUF_HEAPS)		+= qcom_dma_heaps.o
+qcom_dma_heaps-y			:= qcom_dma_heap.o qcom_dt_parser.o qcom_sg_ops.o
+qcom_dma_heaps-$(CONFIG_QCOM_DMABUF_HEAPS_SYSTEM) += qcom_system_heap.o \
+						     qcom_dynamic_page_pool.o
+qcom_dma_heaps-$(CONFIG_QCOM_DMABUF_HEAPS_SYSTEM_SECURE) += qcom_secure_system_heap.o \
+							    qcom_dynamic_page_pool.o \
+							    qcom_dma_heap_secure_utils.o
+qcom_dma_heaps-$(CONFIG_QCOM_DMABUF_HEAPS_CMA)	+= qcom_cma_heap.o
+qcom_dma_heaps-$(CONFIG_QCOM_DMABUF_HEAPS_CARVEOUT) += qcom_carveout_heap.o \
+						       qcom_dma_heap_secure_utils.o
+qcom_dma_heaps-$(CONFIG_QCOM_DMABUF_HEAPS_BITSTREAM_CONTIG) += qcom_bitstream_contig_heap.o

drivers/dma-buf/heaps/qcom_carveout_heap.c

@@ -0,0 +1,589 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * DMA-BUF heap carveout heap allocator. Copied from
+ * drivers/staging/android/ion/heaps/ion_carveout_heap.c as of commit
+ * aeb022cc01ecc ("dma-heap: qcom: Change symbol names to let module be built
+ * in")
+ *
+ * Copyright (C) 2011 Google, Inc.
+ * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
+ */
+
+#include <linux/dma-mapping.h>
+#include <linux/err.h>
+#include <linux/genalloc.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <soc/qcom/secure_buffer.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/list.h>
+#include <linux/dma-buf.h>
+#include <linux/dma-heap.h>
+#include <linux/qcom_dma_heap.h>
+
+#include "qcom_dma_heap_secure_utils.h"
+#include "qcom_sg_ops.h"
+#include "qcom_carveout_heap.h"
+
+#define CARVEOUT_ALLOCATE_FAIL -1
+
+
+/*
+ * @pool_refcount_priv -
+ *	Cookie set by carveout_heap_add_memory for use with its callbacks.
+ *	Cookie provider will call carveout_heap_remove_memory if refcount
+ *	reaches zero.
+ * @pool_refcount_get -
+ *	Function callback to Increase refcount. Returns 0
+ *	on success and fails if refcount is already zero.
+ * @pool_refcount_put - Function callback to decrease refcount.
+ */
+struct carveout_heap {
+	struct dma_heap *heap;
+	struct rw_semaphore mem_sem;
+	struct gen_pool *pool;
+	struct device *dev;
+	bool is_secure;
+	bool is_nomap;
+	phys_addr_t base;
+
+	void *pool_refcount_priv;
+	int (*pool_refcount_get)(void *priv);
+	void (*pool_refcount_put)(void *priv);
+};
+
+struct secure_carveout_heap {
+	u32 token;
+	struct carveout_heap carveout_heap;
+};
+
+static void sc_heap_free(struct qcom_sg_buffer *buffer);
+
+void pages_sync_for_device(struct device *dev, struct page *page,
+			       size_t size, enum dma_data_direction dir)
+{
+	struct scatterlist sg;
+
+	sg_init_table(&sg, 1);
+	sg_set_page(&sg, page, size, 0);
+	/*
+	 * This is not correct - sg_dma_address needs a dma_addr_t that is valid
+	 * for the targeted device, but this works on the currently targeted
+	 * hardware.
+	 */
+	sg_dma_address(&sg) = page_to_phys(page);
+	dma_sync_sg_for_device(dev, &sg, 1, dir);
+}
+
+static int carveout_pool_refcount_get(struct carveout_heap *carveout_heap)
+{
+	if (!carveout_heap->pool_refcount_get)
+		return 0;
+
+	return carveout_heap->pool_refcount_get(carveout_heap->pool_refcount_priv);
+}
+
+static void carveout_pool_refcount_put(struct carveout_heap *carveout_heap)
+{
+	if (!carveout_heap->pool_refcount_put)
+		return;
+
+	carveout_heap->pool_refcount_put(carveout_heap->pool_refcount_priv);
+}
+
+static phys_addr_t carveout_allocate(struct carveout_heap *carveout_heap,
+				     unsigned long size)
+{
+	unsigned long offset = CARVEOUT_ALLOCATE_FAIL;
+
+	down_read(&carveout_heap->mem_sem);
+	if (carveout_heap->pool) {
+		if (carveout_pool_refcount_get(carveout_heap))
+			goto unlock;
+
+		offset = gen_pool_alloc(carveout_heap->pool, size);
+		if (!offset) {
+			offset = CARVEOUT_ALLOCATE_FAIL;
+			carveout_pool_refcount_put(carveout_heap);
+			goto unlock;
+		}
+	}
+
+unlock:
+	up_read(&carveout_heap->mem_sem);
+	return offset;
+}
+
+static void carveout_free(struct carveout_heap *carveout_heap,
+			  phys_addr_t addr, unsigned long size)
+{
+	if (addr == CARVEOUT_ALLOCATE_FAIL)
+		return;
+
+	down_read(&carveout_heap->mem_sem);
+	if (carveout_heap->pool)
+		gen_pool_free(carveout_heap->pool, addr, size);
+	carveout_pool_refcount_put(carveout_heap);
+	up_read(&carveout_heap->mem_sem);
+}
+
+struct mem_buf_vmperm *
+carveout_setup_vmperm(struct carveout_heap *carveout_heap,
+			struct sg_table *sgt)
+{
+	struct secure_carveout_heap *sc_heap;
+	struct mem_buf_vmperm *vmperm;
+	int *vmids, *perms;
+	u32 nr;
+	int ret;
+
+	if (!carveout_heap->is_secure) {
+		vmperm = mem_buf_vmperm_alloc(sgt);
+		return vmperm;
+	}
+
+	sc_heap = container_of(carveout_heap,
+			struct secure_carveout_heap, carveout_heap);
+
+	ret = get_vmperm_from_ion_flags(sc_heap->token,
+			&vmids, &perms, &nr);
+	if (ret)
+		return ERR_PTR(ret);
+
+	vmperm = mem_buf_vmperm_alloc_staticvm(sgt, vmids, perms, nr);
+	kfree(vmids);
+	kfree(perms);
+
+	return vmperm;
+}
+
+static struct dma_buf *__carveout_heap_allocate(struct carveout_heap *carveout_heap,
+						unsigned long len,
+						unsigned long fd_flags,
+						unsigned long heap_flags,
+						void (*buffer_free)(struct qcom_sg_buffer *))
+{
+	struct sg_table *table;
+	struct qcom_sg_buffer *buffer;
+	phys_addr_t paddr;
+	int ret;
+	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
+	struct dma_buf *dmabuf;
+	struct device *dev = carveout_heap->dev;
+
+	buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
+	if (!buffer)
+		return ERR_PTR(-ENOMEM);
+
+	/* Initialize the buffer */
+	INIT_LIST_HEAD(&buffer->attachments);
+	mutex_init(&buffer->lock);
+	buffer->heap = carveout_heap->heap;
+	buffer->len = len;
+	buffer->free = buffer_free;
+	if (carveout_heap->is_nomap)
+		buffer->uncached = true;
+
+	table = &buffer->sg_table;
+	ret = sg_alloc_table(table, 1, GFP_KERNEL);
+	if (ret)
+		goto err_free;
+
+	paddr = carveout_allocate(carveout_heap, len);
+	if (paddr == CARVEOUT_ALLOCATE_FAIL) {
+		ret = -ENOMEM;
+		goto err_free_table;
+	}
+
+	sg_set_page(table->sgl, pfn_to_page(PFN_DOWN(paddr)), len, 0);
+
+	if (!carveout_heap->is_secure && !carveout_heap->is_nomap)
+		pages_sync_for_device(dev, sg_page(table->sgl),
+				      buffer->len, DMA_FROM_DEVICE);
+
+	buffer->vmperm = carveout_setup_vmperm(carveout_heap, &buffer->sg_table);
+	if (IS_ERR(buffer->vmperm))
+		goto err_free_carveout;
+
+
+	/* Instantiate our dma_buf */
+	exp_info.exp_name = dma_heap_get_name(carveout_heap->heap);
+	exp_info.size = buffer->len;
+	exp_info.flags = fd_flags;
+	exp_info.priv = buffer;
+	dmabuf = mem_buf_dma_buf_export(&exp_info, &qcom_sg_buf_ops);
+	if (IS_ERR(dmabuf)) {
+		ret = PTR_ERR(dmabuf);
+		goto err_free_vmperm;
+	}
+
+	return dmabuf;
+
+err_free_vmperm:
+	mem_buf_vmperm_release(buffer->vmperm);
+err_free_carveout:
+	carveout_free(carveout_heap, paddr, len);
+err_free_table:
+	sg_free_table(table);
+err_free:
+	kfree(buffer);
+	return ERR_PTR(ret);
+}
+
+static int carveout_pages_zero(struct page *page, size_t size, pgprot_t prot);
+
+static void carveout_heap_free(struct qcom_sg_buffer *buffer)
+{
+	struct carveout_heap *carveout_heap;
+	struct sg_table *table = &buffer->sg_table;
+	struct page *page = sg_page(table->sgl);
+	phys_addr_t paddr = page_to_phys(page);
+	struct device *dev;
+
+	carveout_heap = dma_heap_get_drvdata(buffer->heap);
+
+	dev = carveout_heap->dev;
+
+	if (!carveout_heap->is_nomap) {
+		carveout_pages_zero(page, buffer->len, PAGE_KERNEL);
+		pages_sync_for_device(dev, page,
+			      buffer->len, DMA_BIDIRECTIONAL);
+	} else {
+		carveout_pages_zero(page, buffer->len, pgprot_writecombine(PAGE_KERNEL));
+	}
+
+	carveout_free(carveout_heap, paddr, buffer->len);
+	sg_free_table(table);
+	kfree(buffer);
+}
+
+
+static struct dma_buf *carveout_heap_allocate(struct dma_heap *heap,
+					      unsigned long len,
+					      unsigned long fd_flags,
+					      unsigned long heap_flags)
+{
+	struct carveout_heap *carveout_heap = dma_heap_get_drvdata(heap);
+
+	return __carveout_heap_allocate(carveout_heap, len, fd_flags,
+					heap_flags, carveout_heap_free);
+}
+
+static int carveout_heap_clear_pages(struct page **pages, int num, pgprot_t prot)
+{
+	void *addr = vmap(pages, num, VM_MAP, prot);
+
+	if (!addr)
+		return -ENOMEM;
+	memset(addr, 0, PAGE_SIZE * num);
+	vunmap(addr);
+
+	return 0;
+}
+
+static int carveout_heap_sglist_zero(struct scatterlist *sgl, unsigned int nents, pgprot_t prot)
+{
+	int p = 0;
+	int ret = 0;
+	struct sg_page_iter piter;
+	struct page *pages[32];
+
+	for_each_sg_page(sgl, &piter, nents, 0) {
+		pages[p++] = sg_page_iter_page(&piter);
+		if (p == ARRAY_SIZE(pages)) {
+			ret = carveout_heap_clear_pages(pages, p, prot);
+			if (ret)
+				return ret;
+			p = 0;
+		}
+	}
+	if (p)
+		ret = carveout_heap_clear_pages(pages, p, prot);
+
+	return ret;
+}
+
+static int carveout_pages_zero(struct page *page, size_t size, pgprot_t prot)
+{
+	struct scatterlist sg;
+
+	sg_init_table(&sg, 1);
+	sg_set_page(&sg, page, size, 0);
+	return carveout_heap_sglist_zero(&sg, 1, prot);
+}
+
+static int carveout_init_heap_memory(struct carveout_heap *co_heap,
+				     phys_addr_t base, ssize_t size,
+				     bool sync)
+{
+	struct page *page = pfn_to_page(PFN_DOWN(base));
+	struct device *dev = co_heap->dev;
+	int ret = 0;
+
+	if (sync) {
+		if (!pfn_valid(PFN_DOWN(base)))
+			return -EINVAL;
+		pages_sync_for_device(dev, page, size, DMA_BIDIRECTIONAL);
+	}
+
+	ret = carveout_pages_zero(page, size, pgprot_writecombine(PAGE_KERNEL));
+	if (ret)
+		return ret;
+
+	co_heap->pool = gen_pool_create(PAGE_SHIFT, -1);
+	if (!co_heap->pool)
+		return -ENOMEM;
+
+	co_heap->base = base;
+	gen_pool_add(co_heap->pool, co_heap->base, size, -1);
+
+	return 0;
+}
+
+int carveout_heap_add_memory(char *heap_name, struct sg_table *sgt, void *cookie,
+			int (*get)(void *), void (*put)(void *))
+{
+	struct dma_heap *heap;
+	struct carveout_heap *carveout_heap;
+	int ret;
+
+	if (!sgt || sgt->nents != 1)
+		return -EINVAL;
+
+	heap = dma_heap_find(heap_name);
+	if (!heap) {
+		pr_err_ratelimited("%s: No heap named %s\n", __func__, heap_name);
+		return -EINVAL;
+	}
+
+	if (!get || !put || !cookie) {
+		pr_err_ratelimited("%s: Missing refcount callbacks\n", __func__);
+		return -EINVAL;
+	}
+
+	carveout_heap = dma_heap_get_drvdata(heap);
+
+	down_write(&carveout_heap->mem_sem);
+	if (carveout_heap->pool) {
+		ret = -EBUSY;
+		goto unlock;
+	}
+
+	ret = carveout_init_heap_memory(carveout_heap,
+					page_to_phys(sg_page(sgt->sgl)),
+					sgt->sgl->length, true);
+	if (ret)
+		goto unlock;
+
+	carveout_heap->pool_refcount_priv = cookie;
+	carveout_heap->pool_refcount_get = get;
+	carveout_heap->pool_refcount_put = put;
+
+unlock:
+	up_write(&carveout_heap->mem_sem);
+	return ret;
+}
+EXPORT_SYMBOL(carveout_heap_add_memory);
+
+int carveout_heap_remove_memory(char *heap_name,
+				struct sg_table *sgt)
+{
+	struct dma_heap *heap;
+	struct carveout_heap *carveout_heap;
+	phys_addr_t base;
+	int ret = 0;
+
+	if (!sgt || sgt->nents != 1)
+		return -EINVAL;
+
+	heap = dma_heap_find(heap_name);
+	if (!heap)
+		return -EINVAL;
+
+	carveout_heap = dma_heap_get_drvdata(heap);
+
+	down_write(&carveout_heap->mem_sem);
+	if (!carveout_heap->pool) {
+		ret = -EINVAL;
+		goto unlock;
+	}
+
+	base = page_to_phys(sg_page(sgt->sgl));
+	if (carveout_heap->base != base) {
+		ret = 0;
+		goto unlock;
+	}
+
+	if (gen_pool_size(carveout_heap->pool) !=
+	    gen_pool_avail(carveout_heap->pool)) {
+		ret = -EBUSY;
+		goto unlock;
+	}
+
+	gen_pool_destroy(carveout_heap->pool);
+	carveout_heap->pool = NULL;
+	carveout_heap->base = 0;
+	carveout_heap->pool_refcount_priv = NULL;
+	carveout_heap->pool_refcount_get = NULL;
+	carveout_heap->pool_refcount_put = NULL;
+unlock:
+	up_write(&carveout_heap->mem_sem);
+	return ret;
+}
+EXPORT_SYMBOL(carveout_heap_remove_memory);
+
+static int __carveout_heap_init(struct platform_heap *heap_data,
+				struct carveout_heap *carveout_heap,
+				bool sync)
+{
+	struct device *dev = heap_data->dev;
+	bool dynamic_heap = heap_data->is_dynamic;
+	int ret = 0;
+
+	carveout_heap->dev = dev;
+	if (!dynamic_heap)
+		ret = carveout_init_heap_memory(carveout_heap,
+						heap_data->base,
+						heap_data->size, sync);
+
+	init_rwsem(&carveout_heap->mem_sem);
+
+	return ret;
+}
+
+static const struct dma_heap_ops carveout_heap_ops = {
+	.allocate = carveout_heap_allocate,
+};
+
+static void carveout_heap_destroy(struct carveout_heap *heap);
+
+int qcom_carveout_heap_create(struct platform_heap *heap_data)
+{
+	struct dma_heap_export_info exp_info;
+	struct carveout_heap *carveout_heap;
+	int ret;
+
+	carveout_heap = kzalloc(sizeof(*carveout_heap), GFP_KERNEL);
+	if (!carveout_heap)
+		return -ENOMEM;
+
+	ret = __carveout_heap_init(heap_data, carveout_heap, !heap_data->is_nomap);
+	if (ret)
+		goto err;
+
+	carveout_heap->is_secure = false;
+	carveout_heap->is_nomap = heap_data->is_nomap;
+
+	exp_info.name = heap_data->name;
+	exp_info.ops = &carveout_heap_ops;
+	exp_info.priv = carveout_heap;
+
+	carveout_heap->heap = dma_heap_add(&exp_info);
+	if (IS_ERR(carveout_heap->heap)) {
+		ret = PTR_ERR(carveout_heap->heap);
+		goto destroy_heap;
+	}
+
+	return 0;
+
+destroy_heap:
+	carveout_heap_destroy(carveout_heap);
+err:
+	kfree(carveout_heap);
+
+	return ret;
+}
+
+static void carveout_heap_destroy(struct carveout_heap *carveout_heap)
+{
+	down_write(&carveout_heap->mem_sem);
+	if (carveout_heap->pool)
+		gen_pool_destroy(carveout_heap->pool);
+	up_write(&carveout_heap->mem_sem);
+	carveout_heap = NULL;
+}
+
+static struct dma_buf *sc_heap_allocate(struct dma_heap *heap,
+					unsigned long len,
+					unsigned long fd_flags,
+					unsigned long heap_flags)
+{
+	struct secure_carveout_heap *sc_heap;
+
+	sc_heap = dma_heap_get_drvdata(heap);
+	return  __carveout_heap_allocate(&sc_heap->carveout_heap, len,
+					 fd_flags, heap_flags, sc_heap_free);
+}
+
+static void sc_heap_free(struct qcom_sg_buffer *buffer)
+{
+	struct secure_carveout_heap *sc_heap;
+	struct sg_table *table = &buffer->sg_table;
+	struct page *page = sg_page(table->sgl);
+	phys_addr_t paddr = PFN_PHYS(page_to_pfn(page));
+
+	sc_heap = dma_heap_get_drvdata(buffer->heap);
+
+	if (qcom_is_buffer_hlos_accessible(sc_heap->token)) {
+		if (!buffer->uncached)
+			carveout_pages_zero(page, buffer->len, PAGE_KERNEL);
+		else
+			carveout_pages_zero(page, buffer->len, pgprot_writecombine(PAGE_KERNEL));
+	}
+	carveout_free(&sc_heap->carveout_heap, paddr, buffer->len);
+	sg_free_table(table);
+	kfree(buffer);
+}
+
+static struct dma_heap_ops sc_heap_ops = {
+	.allocate = sc_heap_allocate,
+};
+
+int qcom_secure_carveout_heap_create(struct platform_heap *heap_data)
+{
+	struct dma_heap_export_info exp_info;
+	struct secure_carveout_heap *sc_heap;
+	int ret;
+
+	sc_heap = kzalloc(sizeof(*sc_heap), GFP_KERNEL);
+	if (!sc_heap)
+		return -ENOMEM;
+
+	ret = __carveout_heap_init(heap_data, &sc_heap->carveout_heap, false);
+	if (ret)
+		goto err;
+
+	ret = hyp_assign_from_flags(heap_data->base, heap_data->size,
+				    heap_data->token);
+	if (ret) {
+		pr_err("secure_carveout_heap: Assign token 0x%x failed\n",
+		       heap_data->token);
+		goto destroy_heap;
+	}
+
+	sc_heap->token = heap_data->token;
+	sc_heap->carveout_heap.is_secure = true;
+
+	exp_info.name = heap_data->name;
+	exp_info.ops = &sc_heap_ops;
+	exp_info.priv = sc_heap;
+
+	sc_heap->carveout_heap.heap = dma_heap_add(&exp_info);
+	if (IS_ERR(sc_heap->carveout_heap.heap)) {
+		ret = PTR_ERR(sc_heap->carveout_heap.heap);
+		goto destroy_heap;
+	}
+
+	return 0;
+
+destroy_heap:
+	carveout_heap_destroy(&sc_heap->carveout_heap);
+err:
+	kfree(sc_heap);
+
+	return ret;
+}

drivers/dma-buf/heaps/qcom_carveout_heap.h

@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
+ */
+
+#ifndef _QCOM_CARVEOUT_HEAP_H
+#define _QCOM_CARVEOUT_HEAP_H
+
+#include "qcom_dt_parser.h"
+
+#ifdef CONFIG_QCOM_DMABUF_HEAPS_CARVEOUT
+int qcom_secure_carveout_heap_create(struct platform_heap *heap_data);
+int qcom_carveout_heap_create(struct platform_heap *heap_data);
+#else
+static int qcom_secure_carveout_heap_create(struct platform_heap *heap_data)
+{
+	return 1;
+}
+static int qcom_carveout_heap_create(struct platform_heap *heap_data)
+{
+	return 1;
+}
+#endif
+
+#endif /* _QCOM_CARVEOUT_HEAP_H */

drivers/dma-buf/heaps/qcom_cma_heap.c

@@ -0,0 +1,198 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * DMABUF CMA heap exporter
+ * Copied from drivers/dma-buf/heaps/cma_heap.c as of commit b61614ec318a
+ * ("dma-buf: heaps: Add CMA heap to dmabuf heaps")
+ *
+ * Copyright (C) 2012, 2019 Linaro Ltd.
+ * Author: <benjamin.gaignard@linaro.org> for ST-Ericsson.
+ * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
+ */
+
+#include <linux/cma.h>
+#include <linux/device.h>
+#include <linux/dma-buf.h>
+#include <linux/dma-heap.h>
+#include <linux/dma-map-ops.h>
+#include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/highmem.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/scatterlist.h>
+#include <linux/sched/signal.h>
+#include <linux/list.h>
+
+#include "qcom_cma_heap.h"
+#include "qcom_sg_ops.h"
+
+struct cma_heap {
+	struct cma *cma;
+	/* max_align is in units of page_order, similar to CONFIG_CMA_ALIGNMENT */
+	u32 max_align;
+	bool uncached;
+};
+
+static void cma_heap_free(struct qcom_sg_buffer *buffer)
+{
+	struct cma_heap *cma_heap;
+	unsigned long nr_pages = buffer->len >> PAGE_SHIFT;
+	struct page *cma_pages = sg_page(buffer->sg_table.sgl);
+
+	cma_heap = dma_heap_get_drvdata(buffer->heap);
+
+	/* free page list */
+	sg_free_table(&buffer->sg_table);
+	/* release memory */
+	cma_release(cma_heap->cma, cma_pages, nr_pages);
+	kfree(buffer);
+}
+
+/* dmabuf heap CMA operations functions */
+struct dma_buf *cma_heap_allocate(struct dma_heap *heap,
+				  unsigned long len,
+				  unsigned long fd_flags,
+				  unsigned long heap_flags)
+{
+	struct cma_heap *cma_heap;
+	struct qcom_sg_buffer *helper_buffer;
+	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
+	struct page *cma_pages;
+	size_t size = PAGE_ALIGN(len);
+	unsigned long nr_pages = size >> PAGE_SHIFT;
+	unsigned long align = get_order(size);
+	struct dma_buf *dmabuf;
+	int ret = -ENOMEM;
+
+	cma_heap = dma_heap_get_drvdata(heap);
+
+	if (align > cma_heap->max_align)
+		align = cma_heap->max_align;
+
+	helper_buffer = kzalloc(sizeof(*helper_buffer), GFP_KERNEL);
+	if (!helper_buffer)
+		return ERR_PTR(-ENOMEM);
+
+	helper_buffer->heap = heap;
+	INIT_LIST_HEAD(&helper_buffer->attachments);
+	mutex_init(&helper_buffer->lock);
+	helper_buffer->len = size;
+	helper_buffer->uncached = cma_heap->uncached;
+	helper_buffer->free = cma_heap_free;
+
+	cma_pages = cma_alloc(cma_heap->cma, nr_pages, align, false);
+	if (!cma_pages)
+		goto free_buf;
+
+	if (PageHighMem(cma_pages)) {
+		unsigned long nr_clear_pages = nr_pages;
+		struct page *page = cma_pages;
+
+		while (nr_clear_pages > 0) {
+			void *vaddr = kmap_atomic(page);
+
+			memset(vaddr, 0, PAGE_SIZE);
+			kunmap_atomic(vaddr);
+			/*
+			 * Avoid wasting time zeroing memory if the process
+			 * has been killed by SIGKILL
+			 */
+			if (fatal_signal_pending(current))
+				goto free_cma;
+
+			page++;
+			nr_clear_pages--;
+		}
+	} else {
+		memset(page_address(cma_pages), 0, size);
+	}
+
+	ret = sg_alloc_table(&helper_buffer->sg_table, 1, GFP_KERNEL);
+	if (ret)
+		goto free_cma;
+
+	sg_set_page(helper_buffer->sg_table.sgl, cma_pages, size, 0);
+
+	helper_buffer->vmperm = mem_buf_vmperm_alloc(&helper_buffer->sg_table);
+	if (IS_ERR(helper_buffer->vmperm))
+		goto free_sgtable;
+
+	if (helper_buffer->uncached) {
+		dma_map_sgtable(dma_heap_get_dev(heap), &helper_buffer->sg_table,
+				DMA_BIDIRECTIONAL, 0);
+		dma_unmap_sgtable(dma_heap_get_dev(heap), &helper_buffer->sg_table,
+				  DMA_BIDIRECTIONAL, 0);
+	}
+
+	/* create the dmabuf */
+	exp_info.exp_name = dma_heap_get_name(heap);
+	exp_info.size = helper_buffer->len;
+	exp_info.flags = fd_flags;
+	exp_info.priv = helper_buffer;
+	dmabuf = mem_buf_dma_buf_export(&exp_info, &qcom_sg_buf_ops);
+	if (IS_ERR(dmabuf)) {
+		ret = PTR_ERR(dmabuf);
+		goto vmperm_release;
+	}
+
+	return dmabuf;
+
+vmperm_release:
+	mem_buf_vmperm_release(helper_buffer->vmperm);
+free_sgtable:
+	sg_free_table(&helper_buffer->sg_table);
+free_cma:
+	cma_release(cma_heap->cma, cma_pages, nr_pages);
+free_buf:
+	kfree(helper_buffer);
+	return ERR_PTR(ret);
+}
+
+static const struct dma_heap_ops cma_heap_ops = {
+	.allocate = cma_heap_allocate,
+};
+
+static int __add_cma_heap(struct platform_heap *heap_data, void *data)
+{
+	struct cma_heap *cma_heap;
+	struct dma_heap_export_info exp_info;
+	struct dma_heap *heap;
+
+	if (!heap_data->dev->cma_area) {
+		pr_err("%s: CMA area for device uninitialized!\n", __func__);
+		return -EINVAL;
+	}
+
+	cma_heap = kzalloc(sizeof(*cma_heap), GFP_KERNEL);
+	if (!cma_heap)
+		return -ENOMEM;
+
+	cma_heap->cma = heap_data->dev->cma_area;
+	cma_heap->max_align = CONFIG_CMA_ALIGNMENT;
+	if (heap_data->max_align)
+		cma_heap->max_align = heap_data->max_align;
+	cma_heap->uncached = heap_data->is_uncached;
+
+	exp_info.name = heap_data->name;
+	exp_info.ops = &cma_heap_ops;
+	exp_info.priv = cma_heap;
+
+	heap = dma_heap_add(&exp_info);
+	if (IS_ERR(heap)) {
+		int ret = PTR_ERR(heap);
+
+		kfree(cma_heap);
+		return ret;
+	}
+
+	if (cma_heap->uncached)
+		dma_coerce_mask_and_coherent(dma_heap_get_dev(heap),
+					     DMA_BIT_MASK(64));
+
+	return 0;
+}
+
+int qcom_add_cma_heap(struct platform_heap *heap_data)
+{
+	return __add_cma_heap(heap_data, NULL);
+}

drivers/dma-buf/heaps/qcom_cma_heap.h

@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
+ */
+
+#ifndef _QCOM_CMA_HEAP_H
+#define _QCOM_CMA_HEAP_H
+
+#include "qcom_dt_parser.h"
+
+#ifdef CONFIG_QCOM_DMABUF_HEAPS_CMA
+int qcom_add_cma_heap(struct platform_heap *heap_data);
+#else
+static int qcom_add_cma_heap(struct platform_heap *heap_data)
+{
+	return 1;
+}
+#endif
+
+#endif /* _QCOM_CMA_HEAP_H */

drivers/dma-buf/heaps/qcom_dma_heap.c

@@ -0,0 +1,98 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
+ */
+
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/err.h>
+
+#include <linux/qcom_dma_heap.h>
+#include "qcom_cma_heap.h"
+#include "qcom_dt_parser.h"
+#include "qcom_system_heap.h"
+#include "qcom_carveout_heap.h"
+#include "qcom_secure_system_heap.h"
+
+static int qcom_dma_heap_probe(struct platform_device *pdev)
+{
+	int ret = 0;
+	int i;
+	struct platform_data *heaps;
+
+	qcom_system_heap_create("qcom,system", "system", false);
+#ifdef CONFIG_QCOM_DMABUF_HEAPS_SYSTEM_UNCACHED
+	qcom_system_heap_create("qcom,system-uncached", NULL, true);
+#endif
+	qcom_secure_system_heap_create("qcom,secure-pixel", NULL,
+				       QCOM_DMA_HEAP_FLAG_CP_PIXEL);
+	qcom_secure_system_heap_create("qcom,secure-non-pixel", NULL,
+				       QCOM_DMA_HEAP_FLAG_CP_NON_PIXEL);
+
+	heaps = parse_heap_dt(pdev);
+	if (IS_ERR_OR_NULL(heaps))
+		return PTR_ERR(heaps);
+
+	for (i = 0; i < heaps->nr; i++) {
+		struct platform_heap *heap_data = &heaps->heaps[i];
+
+		switch (heap_data->type) {
+		case HEAP_TYPE_SECURE_CARVEOUT:
+			ret = qcom_secure_carveout_heap_create(heap_data);
+			if (ret < 0)
+				pr_err("%s: DMA-BUF Heap: Failed to create %s, error is %d\n",
+				       __func__, heap_data->name, ret);
+			else if (!ret)
+				pr_info("%s: DMA-BUF Heap: Created %s\n", __func__,
+					heap_data->name);
+			break;
+		case HEAP_TYPE_CARVEOUT:
+			ret = qcom_carveout_heap_create(heap_data);
+			if (ret < 0)
+				pr_err("%s: DMA-BUF Heap: Failed to create %s, error is %d\n",
+				       __func__, heap_data->name, ret);
+			else if (!ret)
+				pr_info("%s: DMA-BUF Heap: Created %s\n", __func__,
+					heap_data->name);
+			break;
+		case HEAP_TYPE_CMA:
+			ret = qcom_add_cma_heap(heap_data);
+			if (ret < 0)
+				pr_err("%s: DMA-BUF Heap: Failed to create %s, error is %d\n",
+				       __func__, heap_data->name, ret);
+			else if (!ret)
+				pr_info("%s: DMA-BUF Heap: Created %s\n", __func__,
+					heap_data->name);
+			break;
+		default:
+			pr_err("%s: Unknown heap type %u\n", __func__, heap_data->type);
+			break;
+		}
+	}
+
+	free_pdata(heaps);
+
+	return ret;
+}
+
+static const struct of_device_id qcom_dma_heap_match_table[] = {
+	{.compatible = "qcom,dma-heaps"},
+	{},
+};
+
+static struct platform_driver qcom_dma_heap_driver = {
+	.probe = qcom_dma_heap_probe,
+	.driver = {
+		.name = "qcom-dma-heap",
+		.of_match_table = qcom_dma_heap_match_table,
+	},
+};
+
+static int __init init_heap_driver(void)
+{
+	return platform_driver_register(&qcom_dma_heap_driver);
+}
+module_init(init_heap_driver);
+
+MODULE_LICENSE("GPL v2");

drivers/dma-buf/heaps/qcom_dma_heap_priv.h

@@ -0,0 +1,45 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2011 Google, Inc.
+ * Copyright (C) 2019 Linaro Ltd.
+ * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
+ */
+
+#ifndef _QCOM_DMA_HEAP_PRIV_H
+#define _QCOM_DMA_HEAP_PRIV_H
+
+#include <linux/dma-heap.h>
+#include <linux/list.h>
+
+/**
+ * struct heap_helper_buffer - helper buffer metadata
+ * @heap:		back pointer to the heap the buffer came from
+ * @dmabuf:		backing dma-buf for this buffer
+ * @size:		size of the buffer
+ * @priv_virt		pointer to heap specific private value
+ * @lock		mutext to protect the data in this structure
+ * @vmap_cnt		count of vmap references on the buffer
+ * @vaddr		vmap'ed virtual address
+ * @pagecount		number of pages in the buffer
+ * @pages		list of page pointers
+ * @attachments		list of device attachments
+ *
+ * @free		heap callback to free the buffer
+ */
+struct heap_helper_buffer {
+	struct dma_heap *heap;
+	struct dma_buf *dmabuf;
+	size_t size;
+
+	void *priv_virt;
+	struct mutex lock;
+	int vmap_cnt;
+	void *vaddr;
+	pgoff_t pagecount;
+	struct page **pages;
+	struct list_head attachments;
+
+	void (*free)(struct heap_helper_buffer *buffer);
+};
+
+#endif /* _QCOM_DMA_HEAP_PRIV_H */

drivers/dma-buf/heaps/qcom_dma_heap_secure_utils.c

@@ -0,0 +1,272 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
+ */
+
+#include <linux/slab.h>
+#include <soc/qcom/secure_buffer.h>
+#include <linux/qcom_dma_heap.h>
+
+int get_secure_vmid(unsigned long flags)
+{
+	if (flags & QCOM_DMA_HEAP_FLAG_CP_TZ)
+		return VMID_TZ;
+	if (flags & QCOM_DMA_HEAP_FLAG_CP_TOUCH)
+		return VMID_CP_TOUCH;
+	if (flags & QCOM_DMA_HEAP_FLAG_CP_BITSTREAM)
+		return VMID_CP_BITSTREAM;
+	if (flags & QCOM_DMA_HEAP_FLAG_CP_PIXEL)
+		return VMID_CP_PIXEL;
+	if (flags & QCOM_DMA_HEAP_FLAG_CP_NON_PIXEL)
+		return VMID_CP_NON_PIXEL;
+	if (flags & QCOM_DMA_HEAP_FLAG_CP_CAMERA)
+		return VMID_CP_CAMERA;
+	if (flags & QCOM_DMA_HEAP_FLAG_CP_SEC_DISPLAY)
+		return VMID_CP_SEC_DISPLAY;
+	if (flags & QCOM_DMA_HEAP_FLAG_CP_APP)
+		return VMID_CP_APP;
+	if (flags & QCOM_DMA_HEAP_FLAG_CP_CAMERA_PREVIEW)
+		return VMID_CP_CAMERA_PREVIEW;
+	if (flags & QCOM_DMA_HEAP_FLAG_CP_SPSS_SP)
+		return VMID_CP_SPSS_SP;
+	if (flags & QCOM_DMA_HEAP_FLAG_CP_SPSS_SP_SHARED)
+		return VMID_CP_SPSS_SP_SHARED;
+	if (flags & QCOM_DMA_HEAP_FLAG_CP_SPSS_HLOS_SHARED)
+		return VMID_CP_SPSS_HLOS_SHARED;
+	if (flags & QCOM_DMA_HEAP_FLAG_CP_CDSP)
+		return VMID_CP_CDSP;
+	if (flags & QCOM_DMA_HEAP_FLAG_CP_MSS_MSA)
+		return VMID_MSS_MSA;
+
+	return -EINVAL;
+}
+
+static unsigned int count_set_bits(unsigned long val)
+{
+	return ((unsigned int)bitmap_weight(&val, BITS_PER_LONG));
+}
+
+bool qcom_is_buffer_hlos_accessible(unsigned long flags)
+{
+	if (!(flags & QCOM_DMA_HEAP_FLAG_CP_HLOS) &&
+	    !(flags & QCOM_DMA_HEAP_FLAG_CP_SPSS_HLOS_SHARED))
+		return false;
+
+	return true;
+}
+
+static int get_vmid(unsigned long flags)
+{
+	int vmid;
+
+	vmid = get_secure_vmid(flags);
+	if (vmid < 0) {
+		if (flags & QCOM_DMA_HEAP_FLAG_CP_HLOS)
+			vmid = VMID_HLOS;
+	}
+	return vmid;
+}
+
+static int populate_vm_list(unsigned long flags, unsigned int *vm_list,
+			 int nelems)
+{
+	unsigned int itr = 0;
+	int vmid;
+
+	flags = flags & QCOM_DMA_HEAP_FLAGS_CP_MASK;
+	if (!flags)
+		return -EINVAL;
+
+	for_each_set_bit(itr, &flags, BITS_PER_LONG) {
+		vmid = get_vmid(0x1UL << itr);
+		if (vmid < 0 || !nelems)
+			return -EINVAL;
+
+		vm_list[nelems - 1] = vmid;
+		nelems--;
+	}
+	return 0;
+}
+
+static int hyp_unassign_sg(struct sg_table *sgt, int *source_vm_list,
+			   int source_nelems, bool clear_page_private)
+{
+	u32 dest_vmid = VMID_HLOS;
+	u32 dest_perms = PERM_READ | PERM_WRITE | PERM_EXEC;
+	struct scatterlist *sg;
+	int ret, i;
+
+	if (source_nelems <= 0) {
+		pr_err("%s: source_nelems invalid\n",
+		       __func__);
+		ret = -EINVAL;
+		goto out;
+	}
+
+	ret = hyp_assign_table(sgt, source_vm_list, source_nelems, &dest_vmid,
+			       &dest_perms, 1);
+	if (ret)
+		goto out;
+
+	if (clear_page_private)
+		for_each_sg(sgt->sgl, sg, sgt->nents, i)
+			ClearPagePrivate(sg_page(sg));
+out:
+	return ret;
+}
+
+static int hyp_assign_sg(struct sg_table *sgt, int *dest_vm_list,
+			 int dest_nelems, bool set_page_private)
+{
+	u32 source_vmid = VMID_HLOS;
+	struct scatterlist *sg;
+	int *dest_perms;
+	int ret, i;
+
+	if (dest_nelems <= 0) {
+		pr_err("%s: dest_nelems invalid\n",
+		       __func__);
+		ret = -EINVAL;
+		goto out;
+	}
+
+	dest_perms = kcalloc(dest_nelems, sizeof(*dest_perms), GFP_KERNEL);
+	if (!dest_perms) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	for (i = 0; i < dest_nelems; i++)
+		dest_perms[i] = msm_secure_get_vmid_perms(dest_vm_list[i]);
+
+	ret = hyp_assign_table(sgt, &source_vmid, 1,
+			       dest_vm_list, dest_perms, dest_nelems);
+
+	if (ret) {
+		pr_err("%s: Assign call failed\n",
+		       __func__);
+		goto out_free_dest;
+	}
+	if (set_page_private)
+		for_each_sg(sgt->sgl, sg, sgt->nents, i)
+			SetPagePrivate(sg_page(sg));
+
+out_free_dest:
+	kfree(dest_perms);
+out:
+	return ret;
+}
+
+int hyp_unassign_sg_from_flags(struct sg_table *sgt, unsigned long flags,
+			       bool set_page_private)
+{
+	int ret;
+	int *source_vm_list;
+	int source_nelems;
+
+	source_nelems = count_set_bits(flags & QCOM_DMA_HEAP_FLAGS_CP_MASK);
+	source_vm_list = kcalloc(source_nelems, sizeof(*source_vm_list),
+				 GFP_KERNEL);
+	if (!source_vm_list)
+		return -ENOMEM;
+	ret = populate_vm_list(flags, source_vm_list, source_nelems);
+	if (ret) {
+		pr_err("%s: Failed to get secure vmids\n", __func__);
+		goto out_free_source;
+	}
+
+	ret = hyp_unassign_sg(sgt, source_vm_list, source_nelems,
+				  set_page_private);
+
+out_free_source:
+	kfree(source_vm_list);
+	return ret;
+}
+
+int hyp_assign_sg_from_flags(struct sg_table *sgt, unsigned long flags,
+			     bool set_page_private)
+{
+	int ret;
+	int *dest_vm_list = NULL;
+	int dest_nelems;
+
+	dest_nelems = count_set_bits(flags & QCOM_DMA_HEAP_FLAGS_CP_MASK);
+	dest_vm_list = kcalloc(dest_nelems, sizeof(*dest_vm_list), GFP_KERNEL);
+	if (!dest_vm_list) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	ret = populate_vm_list(flags, dest_vm_list, dest_nelems);
+	if (ret) {
+		pr_err("%s: Failed to get secure vmid(s)\n", __func__);
+		goto out_free_dest_vm;
+	}
+
+	ret = hyp_assign_sg(sgt, dest_vm_list, dest_nelems,
+				set_page_private);
+
+out_free_dest_vm:
+	kfree(dest_vm_list);
+out:
+	return ret;
+}
+
+int get_vmperm_from_ion_flags(unsigned long flags, int **_vmids,
+			int **_perms, u32 *_nr)
+{
+	int *vmids, *modes;
+	u32 nr, i;
+
+	if (!_vmids || !_perms || !_nr)
+		return -EINVAL;
+
+	nr = count_set_bits(flags);
+	vmids = kcalloc(nr, sizeof(*vmids), GFP_KERNEL);
+	if (!vmids)
+		return -ENOMEM;
+
+	modes = kcalloc(nr, sizeof(*modes), GFP_KERNEL);
+	if (!modes) {
+		kfree(vmids);
+		return -ENOMEM;
+	}
+
+	if ((flags & ~QCOM_DMA_HEAP_FLAGS_CP_MASK) ||
+	    populate_vm_list(flags, vmids, nr)) {
+		pr_err("%s: Failed to parse secure flags 0x%lx\n", __func__,
+		       flags);
+		kfree(modes);
+		kfree(vmids);
+		return -EINVAL;
+	}
+
+	for (i = 0; i < nr; i++)
+		modes[i] = msm_secure_get_vmid_perms(vmids[i]);
+
+	*_vmids = vmids;
+	*_perms = modes;
+	*_nr = nr;
+	return 0;
+}
+
+int hyp_assign_from_flags(u64 base, u64 size, unsigned long flags)
+{
+	int *vmids, *modes;
+	u32 nr;
+	u32 src_vm = VMID_HLOS;
+	int ret;
+
+	ret = get_vmperm_from_ion_flags(flags, &vmids, &modes, &nr);
+	if (ret)
+		return ret;
+
+	ret = hyp_assign_phys(base, size, &src_vm, 1, vmids, modes, nr);
+	if (ret)
+		pr_err("%s: Assign call failed, flags 0x%lx\n", __func__,
+		       flags);
+
+	kfree(modes);
+	kfree(vmids);
+	return ret;
+}

drivers/dma-buf/heaps/qcom_dma_heap_secure_utils.h

@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
+ */
+
+#ifndef _QCOM_DMA_HEAP_SECURE_UTILS_H
+#define _QCOM_DMA_HEAP_SECURE_UTILS_H
+
+int hyp_assign_sg_from_flags(struct sg_table *sgt, unsigned long flags,
+			     bool set_page_private);
+
+int hyp_unassign_sg_from_flags(struct sg_table *sgt, unsigned long flags,
+			       bool set_page_private);
+
+/*
+ * This function handles conversion from the legacy ion flag based interface
+ * to a precise list of vmids and permissions.
+ */
+int get_vmperm_from_ion_flags(unsigned long flags, int **vmids, int **perms, u32 *nr);
+int hyp_assign_from_flags(u64 base, u64 size, unsigned long flags);
+bool qcom_is_buffer_hlos_accessible(unsigned long flags);
+int get_secure_vmid(unsigned long flags);
+
+#endif /* _QCOM_DMA_HEAP_SECURE_UTILS_H */

drivers/dma-buf/heaps/qcom_dt_parser.c

@@ -0,0 +1,203 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
+ */
+
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/of_platform.h>
+#include <linux/of_address.h>
+#include <linux/of_reserved_mem.h>
+
+#include <linux/qcom_dma_heap.h>
+#include "qcom_dt_parser.h"
+
+#ifdef CONFIG_PANIC_ON_QCOM_DMA_HEAPS_FAILURE
+#define QCOM_DMA_HEAP_WARN(fmt...) panic(fmt)
+#else /* CONFIG_PANIC_ON_QCOM_DMA_HEAPS_FAILURE */
+#define QCOM_DMA_HEAP_WARN(fmt...) WARN(1, fmt)
+#endif /* CONFIG_PANIC_ON_QCOM_DMA_HEAPS_FAILURE */
+
+static int populate_heap(struct device_node *node,
+			 struct platform_heap *heap)
+{
+	int ret;
+
+	/* Mandatory properties */
+	ret = of_property_read_string(node, "qcom,dma-heap-name", &heap->name);
+	if (ret)
+		goto err;
+
+	ret = of_property_read_u32(node, "qcom,dma-heap-type", &heap->type);
+	if (ret) {
+		pr_err("Reading %s property in node %s failed with err %d.\n",
+		       "qcom,dma-heap-type", of_node_full_name(node), ret);
+		goto err;
+	}
+
+	/* Optional properties */
+	heap->is_dynamic = of_property_read_bool(node, "qcom,dynamic-heap");
+
+	heap->is_uncached = of_property_read_bool(node, "qcom,uncached-heap");
+
+	ret = of_property_read_u32(node, "qcom,token", &heap->token);
+	if (ret && ret != -EINVAL)
+		goto err;
+
+	ret = of_property_read_u32(node, "qcom,max-align", &heap->max_align);
+	if (ret && ret != -EINVAL)
+		goto err;
+
+	return 0;
+
+err:
+	if (ret)
+		QCOM_DMA_HEAP_WARN("%s: Unable to populate heap %s, err: %d\n",
+				   __func__, of_node_full_name(node), ret);
+	return ret;
+}
+
+void free_pdata(const struct platform_data *pdata)
+{
+	kfree(pdata->heaps);
+	kfree(pdata);
+}
+
+static int heap_dt_init(struct device_node *mem_node,
+			struct platform_heap *heap)
+{
+	const __be32 *basep;
+	u64 base, size;
+	struct device *dev = heap->dev;
+	struct reserved_mem *rmem;
+	int ret = 0;
+
+
+	rmem = of_reserved_mem_lookup(mem_node);
+
+	if (!rmem) {
+		dev_err(dev, "Failed to find reserved memory region\n");
+		return -EINVAL;
+	}
+
+	/*
+	 * We only need to call this when the memory-region is managed by
+	 * a reserved memory region driver (e.g. CMA, coherent, etc). In that
+	 * case, they will have ops for device specific initialization for
+	 * the memory region. Otherwise, we have a pure carveout, which needs
+	 * not be initialized.
+	 */
+	if (rmem->ops) {
+		ret = of_reserved_mem_device_init_by_idx(dev, dev->of_node, 0);
+		if (ret) {
+			dev_err(dev,
+				"Failed to initialize memory region rc: %d\n",
+				ret);
+			return ret;
+		}
+	}
+
+	basep = of_get_address(mem_node, 0, &size, NULL);
+	if (basep) {
+		base = of_translate_address(mem_node, basep);
+		if (base != OF_BAD_ADDR) {
+			heap->base = base;
+			heap->size = size;
+		} else {
+			ret = -EINVAL;
+			dev_err(heap->dev,
+				"Failed to get heap base/size\n");
+			of_reserved_mem_device_release(dev);
+		}
+	}
+	heap->is_nomap =  of_property_read_bool(mem_node, "no-map");
+
+	return ret;
+}
+
+static void release_reserved_memory_regions(struct platform_heap *heaps,
+					    int idx)
+{
+	struct device *dev;
+	struct device_node *node, *mem_node;
+
+	for (idx = idx - 1; idx >= 0; idx--) {
+		dev = heaps[idx].dev;
+		node = dev->of_node;
+		mem_node = of_parse_phandle(node, "memory-region", 0);
+
+		if (mem_node)
+			of_reserved_mem_device_release(dev);
+
+		of_node_put(mem_node);
+	}
+}
+
+struct platform_data *parse_heap_dt(struct platform_device *pdev)
+{
+	struct platform_data *pdata = NULL;
+	struct device_node *node;
+	struct device_node *mem_node;
+	struct platform_device *new_dev = NULL;
+	const struct device_node *dt_node = pdev->dev.of_node;
+	int ret;
+	int idx = 0;
+
+	pdata = kzalloc(sizeof(*pdata), GFP_KERNEL);
+	if (!pdata)
+		return ERR_PTR(-ENOMEM);
+
+	for_each_available_child_of_node(dt_node, node)
+		pdata->nr++;
+
+	/*
+	 * No heaps defined in the devicetree. However, there may be other
+	 * heaps (e.g. system heaps) that do not need to be defined in the
+	 * devicetree.
+	 */
+	if (!pdata->nr)
+		goto out;
+
+	pdata->heaps = kcalloc(pdata->nr, sizeof(*pdata->heaps), GFP_KERNEL);
+	if (!pdata->heaps) {
+		kfree(pdata);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	for_each_available_child_of_node(dt_node, node) {
+		new_dev = of_platform_device_create(node, NULL, &pdev->dev);
+		if (!new_dev) {
+			pr_err("Failed to create device %s\n", node->name);
+			ret = -EINVAL;
+			goto free_heaps;
+		}
+		of_dma_configure(&new_dev->dev, node, true);
+
+		pdata->heaps[idx].dev = &new_dev->dev;
+
+		ret = populate_heap(node, &pdata->heaps[idx]);
+		if (ret)
+			goto free_heaps;
+
+		mem_node = of_parse_phandle(node, "memory-region", 0);
+		if (mem_node) {
+			ret = heap_dt_init(mem_node, &pdata->heaps[idx]);
+			if (ret)
+				goto free_heaps;
+
+			of_node_put(mem_node);
+		}
+
+		++idx;
+	}
+
+out:
+	return pdata;
+
+free_heaps:
+	of_node_put(node);
+	release_reserved_memory_regions(pdata->heaps, idx);
+	free_pdata(pdata);
+	return ERR_PTR(ret);
+}

drivers/dma-buf/heaps/qcom_dt_parser.h

@@ -0,0 +1,57 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
+ */
+
+#ifndef _QCOM_DT_PARSER_H
+#define _QCOM_DT_PARSER_H
+
+#include <linux/device.h>
+#include <linux/platform_device.h>
+
+/**
+ * struct platform_heap - defines a heap in the given platform
+ * @type:	type of the heap
+ * @name:	used for debug purposes
+ * @base:	base address of heap in physical memory if applicable
+ * @size:	size of the heap in bytes if applicable
+ * @dev:	the device associated with the heap's DT node
+ * @is_dynamic:	indicates if memory can be added or removed from carveout heaps
+ * is_nomap:	indicates if this heap is assosicated with a memory region which is
+ *		which has no-map property.
+ * is_uncached:	indicates if this heap provides uncached memory or not
+ * @token:	the end points to which memory for secure carveout memory is
+ *		assigned to
+ * @max_align:  page order of the maximum alignment. Used by cma heap.
+ *
+ * Provided by the board file.
+ */
+struct platform_heap {
+	u32 type;
+	const char *name;
+	phys_addr_t base;
+	size_t size;
+	struct device *dev;
+	bool is_uncached;
+	bool is_dynamic;
+	bool is_nomap;
+	u32 token;
+	u32 max_align;
+};
+
+/**
+ * struct platform_data - array of platform heaps passed from board file
+ * @nr:    number of structures in the array
+ * @heaps: array of platform_heap structions
+ *
+ * Provided by the board file in the form of platform data to a platform device.
+ */
+struct platform_data {
+	int nr;
+	struct platform_heap *heaps;
+};
+
+struct platform_data *parse_heap_dt(struct platform_device *pdev);
+void free_pdata(const struct platform_data *pdata);
+
+#endif /* _QCOM_DT_PARSER_H */

drivers/dma-buf/heaps/qcom_dynamic_page_pool.c

@@ -0,0 +1,332 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Dynamic page pool system
+ *
+ * Taken from:
+ * mm/dynamic_page_pool.c
+ * Git-repo: https://git.linaro.org/people/john.stultz/android-dev.git
+ * Branch: dma-buf-heap-perf
+ * Git-commit: 458ea8030852755867bdc0384aa40f97aba7a572
+ *
+ * Based on the ION page pool code
+ * Copyright (C) 2011 Google, Inc.
+ *
+ * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
+ */
+
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/swap.h>
+#include <linux/sched/signal.h>
+#include <uapi/linux/sched/types.h>
+
+#include "qcom_dynamic_page_pool.h"
+
+static LIST_HEAD(pool_list);
+static DEFINE_MUTEX(pool_list_lock);
+
+void dynamic_page_pool_add(struct dynamic_page_pool *pool, struct page *page)
+{
+	mutex_lock(&pool->mutex);
+	if (PageHighMem(page)) {
+		list_add_tail(&page->lru, &pool->high_items);
+		pool->high_count++;
+	} else {
+		list_add_tail(&page->lru, &pool->low_items);
+		pool->low_count++;
+	}
+
+	atomic_inc(&pool->count);
+	mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE,
+			    1 << pool->order);
+	mutex_unlock(&pool->mutex);
+}
+
+struct page *dynamic_page_pool_remove(struct dynamic_page_pool *pool, bool high)
+{
+	struct page *page;
+
+	if (high) {
+		BUG_ON(!pool->high_count);
+		page = list_first_entry(&pool->high_items, struct page, lru);
+		pool->high_count--;
+	} else {
+		BUG_ON(!pool->low_count);
+		page = list_first_entry(&pool->low_items, struct page, lru);
+		pool->low_count--;
+	}
+
+	atomic_dec(&pool->count);
+	list_del(&page->lru);
+	mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE,
+			    -(1 << pool->order));
+	return page;
+}
+
+void dynamic_page_pool_free(struct dynamic_page_pool *pool, struct page *page)
+{
+	BUG_ON(pool->order != compound_order(page));
+
+	dynamic_page_pool_add(pool, page);
+}
+
+int dynamic_page_pool_total(struct dynamic_page_pool *pool, bool high)
+{
+	int count = pool->low_count;
+
+	if (high)
+		count += pool->high_count;
+
+	return count << pool->order;
+}
+
+struct dynamic_page_pool *dynamic_page_pool_create(gfp_t gfp_mask, unsigned int order)
+{
+	struct dynamic_page_pool *pool = kmalloc(sizeof(*pool), GFP_KERNEL);
+
+	if (!pool)
+		return NULL;
+	pool->high_count = 0;
+	pool->low_count = 0;
+	INIT_LIST_HEAD(&pool->low_items);
+	INIT_LIST_HEAD(&pool->high_items);
+	pool->gfp_mask = gfp_mask | __GFP_COMP;
+	pool->order = order;
+	mutex_init(&pool->mutex);
+
+	mutex_lock(&pool_list_lock);
+	list_add(&pool->list, &pool_list);
+	mutex_unlock(&pool_list_lock);
+
+	return pool;
+}
+
+void dynamic_page_pool_destroy(struct dynamic_page_pool *pool)
+{
+	struct page *page, *tmp;
+	LIST_HEAD(pages);
+	int num_pages = 0;
+	int ret = DYNAMIC_POOL_SUCCESS;
+
+	/* Remove us from the pool list */
+	mutex_lock(&pool_list_lock);
+	list_del(&pool->list);
+	mutex_unlock(&pool_list_lock);
+
+	/* Free any remaining pages in the pool */
+	mutex_lock(&pool->mutex);
+	while (true) {
+		if (pool->low_count)
+			page = dynamic_page_pool_remove(pool, false);
+		else if (pool->high_count)
+			page = dynamic_page_pool_remove(pool, true);
+		else
+			break;
+
+		list_add(&page->lru, &pages);
+		num_pages++;
+	}
+	mutex_unlock(&pool->mutex);
+
+	if (num_pages && pool->prerelease_callback)
+		ret = pool->prerelease_callback(pool, &pages, num_pages);
+
+	if (ret != DYNAMIC_POOL_SUCCESS) {
+		pr_err("Failed to reclaim pages when destroying the pool!\n");
+		return;
+	}
+
+	list_for_each_entry_safe(page, tmp, &pages, lru) {
+		list_del(&page->lru);
+		__free_pages(page, pool->order);
+	}
+
+	kfree(pool);
+}
+
+static int dynamic_page_pool_do_shrink(struct dynamic_page_pool *pool, gfp_t gfp_mask,
+				       int nr_to_scan)
+{
+	int freed = 0;
+	bool high;
+	struct page *page, *tmp;
+	LIST_HEAD(pages);
+	int ret = DYNAMIC_POOL_SUCCESS;
+
+	if (current_is_kswapd())
+		high = true;
+	else
+		high = !!(gfp_mask & __GFP_HIGHMEM);
+
+	if (nr_to_scan == 0)
+		return dynamic_page_pool_total(pool, high);
+
+	while (freed < nr_to_scan) {
+		mutex_lock(&pool->mutex);
+		if (pool->low_count) {
+			page = dynamic_page_pool_remove(pool, false);
+		} else if (high && pool->high_count) {
+			page = dynamic_page_pool_remove(pool, true);
+		} else {
+			mutex_unlock(&pool->mutex);
+			break;
+		}
+		mutex_unlock(&pool->mutex);
+		list_add(&page->lru, &pages);
+		freed += (1 << pool->order);
+	}
+
+	if (freed && pool->prerelease_callback)
+		ret = pool->prerelease_callback(pool, &pages, freed >> pool->order);
+
+	if (ret != DYNAMIC_POOL_SUCCESS) {
+		pr_err("Failed to reclaim secure page pool pages!\n");
+		return 0;
+	}
+
+	list_for_each_entry_safe(page, tmp, &pages, lru) {
+		list_del(&page->lru);
+		__free_pages(page, pool->order);
+	}
+
+	return freed;
+}
+
+void dynamic_page_pool_shrink_high_and_low(struct dynamic_page_pool **pools_list,
+					   int num_pools, int nr_to_scan)
+{
+	int i, remaining = nr_to_scan;
+
+	mutex_lock(&pool_list_lock);
+	for (i = 0; i < num_pools; i++) {
+		remaining -= dynamic_page_pool_do_shrink(pools_list[i], __GFP_HIGHMEM,
+							 remaining);
+
+		if (remaining <= 0) {
+			mutex_unlock(&pool_list_lock);
+			return;
+		}
+	}
+	mutex_unlock(&pool_list_lock);
+}
+
+static int dynamic_page_pool_shrink(gfp_t gfp_mask, int nr_to_scan)
+{
+	struct dynamic_page_pool *pool;
+	int nr_total = 0;
+	int nr_freed;
+	int only_scan = 0;
+
+	if (!nr_to_scan)
+		only_scan = 1;
+
+	mutex_lock(&pool_list_lock);
+	list_for_each_entry(pool, &pool_list, list) {
+		if (only_scan) {
+			nr_total += dynamic_page_pool_do_shrink(pool,
+								gfp_mask,
+								nr_to_scan);
+		} else {
+			nr_freed = dynamic_page_pool_do_shrink(pool,
+							       gfp_mask,
+							       nr_to_scan);
+			nr_to_scan -= nr_freed;
+			nr_total += nr_freed;
+			if (nr_to_scan <= 0)
+				break;
+		}
+	}
+	mutex_unlock(&pool_list_lock);
+
+	return nr_total;
+}
+
+static unsigned long dynamic_page_pool_shrink_count(struct shrinker *shrinker,
+						    struct shrink_control *sc)
+{
+	return dynamic_page_pool_shrink(sc->gfp_mask, 0);
+}
+
+static unsigned long dynamic_page_pool_shrink_scan(struct shrinker *shrinker,
+						   struct shrink_control *sc)
+{
+	int to_scan = sc->nr_to_scan;
+
+	if (to_scan == 0)
+		return 0;
+
+	return dynamic_page_pool_shrink(sc->gfp_mask, to_scan);
+}
+
+struct dynamic_page_pool **dynamic_page_pool_create_pools(int vmid,
+							  prerelease_callback callback)
+{
+	struct dynamic_page_pool **pool_list;
+	int i;
+	int ret;
+
+	pool_list = kmalloc_array(NUM_ORDERS, sizeof(*pool_list), GFP_KERNEL);
+	if (!pool_list)
+		return ERR_PTR(-ENOMEM);
+
+	for (i = 0; i < NUM_ORDERS; i++) {
+		pool_list[i] = dynamic_page_pool_create(order_flags[i],
+							orders[i]);
+		pool_list[i]->vmid = vmid;
+		pool_list[i]->prerelease_callback = callback;
+		atomic_set(&pool_list[i]->count, 0);
+		pool_list[i]->last_low_watermark_ktime = 0;
+
+		if (IS_ERR_OR_NULL(pool_list[i])) {
+			int j;
+
+			pr_err("%s: page pool creation failed for the order %u pool!\n",
+			       __func__, orders[i]);
+			for (j = 0; j < i; j++)
+				dynamic_page_pool_destroy(pool_list[j]);
+
+			ret = -ENOMEM;
+			goto free_pool_arr;
+		}
+	}
+
+	return pool_list;
+
+free_pool_arr:
+	kfree(pool_list);
+
+	return ERR_PTR(ret);
+}
+
+void dynamic_page_pool_release_pools(struct dynamic_page_pool **pool_list)
+{
+	int i;
+
+	for (i = 0; i < NUM_ORDERS; i++)
+		dynamic_page_pool_destroy(pool_list[i]);
+
+	kfree(pool_list);
+}
+
+static struct shrinker pool_shrinker = {
+	.count_objects = dynamic_page_pool_shrink_count,
+	.scan_objects = dynamic_page_pool_shrink_scan,
+	.seeks = DEFAULT_SEEKS,
+	.batch = 0,
+};
+
+int dynamic_page_pool_init_shrinker(void)
+{
+	int ret;
+	static bool registered;
+
+	if (registered)
+		return 0;
+
+	ret = register_shrinker(&pool_shrinker);
+	if (ret)
+		return ret;
+
+	registered = true;
+	return 0;
+}

drivers/dma-buf/heaps/qcom_dynamic_page_pool.h

@@ -0,0 +1,110 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * ION Memory Allocator kernel interface header
+ *
+ * Taken from:
+ * include/linux/dynamic_page_pool.h
+ * Git-repo: https://git.linaro.org/people/john.stultz/android-dev.git
+ * Branch: dma-buf-heap-perf
+ * Git-commit: 458ea8030852755867bdc0384aa40f97aba7a572
+ *
+ * Copyright (C) 2011 Google, Inc.
+ *
+ * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
+ */
+
+#ifndef _DYN_PAGE_POOL_H
+#define _DYN_PAGE_POOL_H
+
+#include <linux/device.h>
+#include <linux/kref.h>
+#include <linux/mm_types.h>
+#include <linux/mutex.h>
+#include <linux/shrinker.h>
+#include <linux/types.h>
+
+#define HIGH_ORDER_GFP  (((GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN \
+				| __GFP_NORETRY) & ~__GFP_RECLAIM) \
+				| __GFP_COMP)
+#define LOW_ORDER_GFP (GFP_HIGHUSER | __GFP_ZERO | __GFP_COMP)
+
+/*
+ * We add __GFP_NOWARN for order 4 allocations since the core mm/ framework
+ * makes no guarantee of these allocations succeeding.
+ */
+static gfp_t order_flags[] = {HIGH_ORDER_GFP, HIGH_ORDER_GFP,
+			      LOW_ORDER_GFP};
+#if defined(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) && !defined(CONFIG_64BIT) && !defined(CONFIG_ARM_LPAE)
+static const unsigned int orders[] = {8, 4, 0};
+#else
+static const unsigned int orders[] = {9, 4, 0};
+#endif
+#define NUM_ORDERS ARRAY_SIZE(orders)
+
+enum dynamic_pool_callback_ret {
+	DYNAMIC_POOL_SUCCESS,
+	DYNAMIC_POOL_FAILURE,
+};
+
+struct dynamic_page_pool;
+
+typedef enum dynamic_pool_callback_ret (*prerelease_callback)(struct dynamic_page_pool *pool,
+							      struct list_head *pages,
+							      int num_pages);
+
+/**
+ * struct dynamic_page_pool - pagepool struct
+ * @high_count:			number of highmem items in the pool
+ * @low_count:			number of lowmem items in the pool
+ * @count:			total number of pages/items in the pool
+ * @high_items:			list of highmem items
+ * @low_items:			list of lowmem items
+ * @last_low_watermark_ktime:	most recent time at which the zone watermarks were
+ *				low
+ * @refill_worker:		kthread used to refill a pool, if applicable
+ * @mutex:			lock protecting this struct and especially the count
+ *				item list
+ * @gfp_mask:			gfp_mask to use from alloc
+ * @order:			order of pages in the pool
+ * @list:			list node for list of pools
+ * @vmid:			the vmid used for this pool
+ * @prerelease_callback:	preprocessing function called before pages are
+ *				released to buddy
+ *
+ * Allows you to keep a pool of pre allocated pages to use
+ * Keeping a pool of pages that is ready for dma, ie any cached mapping have
+ * been invalidated from the cache, provides a significant performance benefit
+ * on many systems
+ */
+struct dynamic_page_pool {
+	int high_count;
+	int low_count;
+	atomic_t count;
+	struct list_head high_items;
+	struct list_head low_items;
+	ktime_t last_low_watermark_ktime;
+	struct task_struct *refill_worker;
+	struct mutex mutex;
+	gfp_t gfp_mask;
+	unsigned int order;
+	struct list_head list;
+	int vmid;
+	prerelease_callback prerelease_callback;
+};
+
+struct dynamic_page_pool **dynamic_page_pool_create_pools(int vmid,
+							  prerelease_callback callback);
+void dynamic_page_pool_release_pools(struct dynamic_page_pool **pool_list);
+
+struct page *dynamic_page_pool_alloc(struct dynamic_page_pool *pool);
+void dynamic_page_pool_free(struct dynamic_page_pool *pool, struct page *page);
+
+int dynamic_page_pool_init_shrinker(void);
+void dynamic_page_pool_shrink_high_and_low(struct dynamic_page_pool **pools_list,
+					   int num_pools, int nr_to_scan);
+int dynamic_page_pool_total(struct dynamic_page_pool *pool, bool high);
+
+struct page *dynamic_page_pool_remove(struct dynamic_page_pool *pool, bool high);
+void dynamic_page_pool_add(struct dynamic_page_pool *pool, struct page *page);
+
+#endif /* _DYN_PAGE_POOL_H */

drivers/dma-buf/heaps/qcom_secure_system_heap.c

@@ -0,0 +1,716 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * DMABUF System heap exporter
+ * Originally copied from: drivers/dma-buf/heaps/system_heap.c as of commit
+ * 263e38f82cbb ("dma-buf: heaps: Remove redundant heap identifier from system
+ * heap name")
+ *
+ * Additions taken from modifications to drivers/dma-buf/heaps/system-heap.c,
+ * from patches submitted, are listed below:
+ *
+ * Addition that modifies dma_buf ops to use SG tables taken from
+ * drivers/dma-buf/heaps/system-heap.c in:
+ * https://lore.kernel.org/lkml/20201017013255.43568-2-john.stultz@linaro.org/
+ *
+ * Addition that skips unneeded syncs in the dma_buf ops taken from
+ * https://lore.kernel.org/lkml/20201017013255.43568-5-john.stultz@linaro.org/
+ *
+ * Addition that tries to allocate higher order pages taken from
+ * https://lore.kernel.org/lkml/20201017013255.43568-6-john.stultz@linaro.org/
+ *
+ * Addition that implements an uncached heap taken from
+ * https://lore.kernel.org/lkml/20201017013255.43568-8-john.stultz@linaro.org/,
+ * with our own modificaitons made to account for core kernel changes that are
+ * a part of the patch series.
+ *
+ * Pooling functionality taken from:
+ * Git-repo: https://git.linaro.org/people/john.stultz/android-dev.git
+ * Branch: dma-buf-heap-perf
+ * Git-commit: 6f080eb67dce63c6efa57ef564ca4cd762ccebb0
+ * Git-commit: 6fb9593b928c4cb485bef4e88c59c6b9fdf11352
+ *
+ * Branched off from qcom_system_heap.c as of commit a4e135a8e482
+ * ("dt-bindings: ipcc: Add WPSS client to IPCC header") to accommodate secure
+ * pooling.
+ *
+ * Copyright (C) 2011 Google, Inc.
+ * Copyright (C) 2019, 2020 Linaro Ltd.
+ *
+ * Portions based off of Andrew Davis' SRAM heap:
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ *	Andrew F. Davis <afd@ti.com>
+ *
+ * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
+ */
+
+#include <linux/dma-buf.h>
+#include <linux/dma-mapping.h>
+#include <linux/dma-heap.h>
+#include <linux/err.h>
+#include <linux/highmem.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/qcom_dma_heap.h>
+
+#include "qcom_dma_heap_secure_utils.h"
+#include "qcom_dynamic_page_pool.h"
+#include "qcom_sg_ops.h"
+#include "qcom_secure_system_heap.h"
+#include "qcom_system_heap.h"
+
+#define MAX_NR_PREFETCH_REGIONS 32
+
+/*
+ * The video client may not hold the last reference count on the
+ * ion_buffer(s). Delay for a short time after the video client sends
+ * the IOC_DRAIN event to increase the chance that the reference
+ * count drops to zero. Time in milliseconds.
+ */
+#define SHRINK_DELAY 1000
+
+struct prefetch_info {
+	struct list_head list;
+	struct dma_heap *heap;
+	u64 size;
+	bool shrink;
+};
+
+static LIST_HEAD(prefetch_list);
+static DEFINE_SPINLOCK(work_lock);
+static struct delayed_work prefetch_work;
+static struct workqueue_struct *prefetch_wq;
+
+static LIST_HEAD(secure_heaps);
+
+static inline struct dynamic_page_pool **get_sys_heap_page_pool(void)
+{
+	static struct dynamic_page_pool **qcom_sys_heap_pools;
+	struct dma_heap *dma_heap;
+	struct qcom_system_heap *sys_heap;
+
+	if (qcom_sys_heap_pools)
+		return qcom_sys_heap_pools;
+
+	dma_heap = dma_heap_find("qcom,system");
+	if (!dma_heap) {
+		pr_err("Unable to find the system heap\n");
+		goto out;
+	}
+
+	sys_heap = (struct qcom_system_heap *) dma_heap_get_drvdata(dma_heap);
+	qcom_sys_heap_pools = sys_heap->pool_list;
+
+out:
+	return qcom_sys_heap_pools;
+}
+
+static struct sg_table *get_pages(u64 size, struct dma_heap *heap)
+{
+	LIST_HEAD(pages);
+	struct dynamic_page_pool **qcom_sys_heap_pools;
+	unsigned long size_remaining = size;
+	struct page *page, *tmp_page;
+	int max_order = orders[0];
+	int num_pages = 0;
+	struct sg_table *sgt;
+	struct scatterlist *sg;
+	int ret;
+
+	qcom_sys_heap_pools = get_sys_heap_page_pool();
+	if (!qcom_sys_heap_pools) {
+		pr_err("%s: Couldn't obtain the pools for the system heap!\n", __func__);
+		return ERR_PTR(-EINVAL);
+	}
+
+	while (size_remaining > 0) {
+		page = qcom_sys_heap_alloc_largest_available(qcom_sys_heap_pools,
+							     size_remaining,
+							     max_order);
+
+		if (!page) {
+			pr_err("%s: Failed to get pages from the system heap: %d, %d!\n",
+			       __func__, size_remaining, size);
+			ret = -ENOMEM;
+			goto err;
+		}
+
+		list_add_tail(&page->lru, &pages);
+		size_remaining -= page_size(page);
+		max_order = compound_order(page);
+		num_pages++;
+	}
+
+	sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
+	if (!sgt) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	ret = sg_alloc_table(sgt, num_pages, GFP_KERNEL);
+	if (ret) {
+		pr_err("%s: sg table allocation failed with %d\n", __func__, ret);
+		goto free_table;
+	}
+
+	sg = sgt->sgl;
+	list_for_each_entry(page, &pages, lru) {
+		sg_set_page(sg, page, page_size(page), 0);
+		sg = sg_next(sg);
+	}
+
+	dma_map_sgtable(dma_heap_get_dev(heap), sgt, DMA_BIDIRECTIONAL, 0);
+	dma_unmap_sgtable(dma_heap_get_dev(heap), sgt, DMA_BIDIRECTIONAL, 0);
+
+	return sgt;
+
+free_table:
+	kfree(sgt);
+err:
+	list_for_each_entry_safe(page, tmp_page, &pages, lru)
+		__free_pages(page, compound_order(page));
+
+	return ERR_PTR(ret);
+}
+
+static void process_one_prefetch(struct prefetch_info *info)
+{
+	struct qcom_secure_system_heap *secure_heap = dma_heap_get_drvdata(info->heap);
+
+	struct sg_table *sgt;
+	struct scatterlist *sg;
+	struct page *page;
+	int ret;
+	int i, j;
+
+	sgt = get_pages(info->size, info->heap);
+	if (IS_ERR(sgt))
+		return;
+
+	ret = hyp_assign_sg_from_flags(sgt, secure_heap->vmid, true);
+	if (ret)
+		goto err;
+
+	for_each_sgtable_sg(sgt, sg, i) {
+		page = sg_page(sg);
+
+		for (j = 0; j < NUM_ORDERS; j++) {
+			if (compound_order(page) == orders[j])
+				break;
+		}
+		dynamic_page_pool_free(secure_heap->pool_list[j], page);
+	}
+
+	sg_free_table(sgt);
+
+	return;
+
+err:
+	for_each_sgtable_sg(sgt, sg, i) {
+		page = sg_page(sg);
+		__free_pages(page, compound_order(page));
+	}
+	sg_free_table(sgt);
+}
+
+static void process_one_shrink(struct prefetch_info *info)
+{
+	struct qcom_secure_system_heap *secure_heap = dma_heap_get_drvdata(info->heap);
+
+	dynamic_page_pool_shrink_high_and_low(secure_heap->pool_list,
+					      NUM_ORDERS, info->size / PAGE_SIZE);
+}
+
+static void secure_system_heap_prefetch_work(struct work_struct *work)
+{
+	struct prefetch_info *info, *tmp;
+	unsigned long flags;
+
+	spin_lock_irqsave(&work_lock, flags);
+	list_for_each_entry_safe(info, tmp, &prefetch_list, list) {
+		list_del(&info->list);
+		spin_unlock_irqrestore(&work_lock, flags);
+
+		if (info->shrink)
+			process_one_shrink(info);
+		else
+			process_one_prefetch(info);
+
+		kfree(info);
+		spin_lock_irqsave(&work_lock, flags);
+	}
+	spin_unlock_irqrestore(&work_lock, flags);
+}
+
+static int alloc_prefetch_info(struct dma_buf_heap_prefetch_region *region,
+			       bool shrink, struct list_head *items)
+{
+	struct prefetch_info *info;
+
+	info = kmalloc(sizeof(*info), GFP_KERNEL);
+	if (!info)
+		return -ENOMEM;
+
+	info->size = PAGE_ALIGN(region->size);
+	info->heap = region->heap;
+	info->shrink = shrink;
+	INIT_LIST_HEAD(&info->list);
+	list_add_tail(&info->list, items);
+	return 0;
+}
+
+static int __qcom_secure_system_heap_resize(struct dma_buf_heap_prefetch_region *regions,
+					    size_t nr_regions,
+					    bool shrink)
+{
+	struct qcom_secure_system_heap *list_ptr, *secure_heap = NULL;
+	int i, ret = 0;
+	struct prefetch_info *info, *tmp;
+	unsigned long flags;
+	LIST_HEAD(items);
+
+	if (!regions || nr_regions > MAX_NR_PREFETCH_REGIONS) {
+		pr_err("%s: Invalid input for %s\n", __func__,
+		       (shrink) ? "drain" : "prefetch");
+		return -EINVAL;
+	}
+
+	for (i = 0; i < nr_regions; i++) {
+		list_for_each_entry(list_ptr, &secure_heaps, list) {
+			if (list_ptr == dma_heap_get_drvdata(regions->heap)) {
+				secure_heap = list_ptr;
+				break;
+			}
+		}
+
+		if (!secure_heap) {
+			pr_err("%s: %s is not a secure system heap!\n", __func__,
+			       dma_heap_get_name(regions->heap));
+			ret = -EINVAL;
+			goto out_free;
+		}
+
+		ret = alloc_prefetch_info(&regions[i], shrink, &items);
+		if (ret)
+			goto out_free;
+	}
+
+	spin_lock_irqsave(&work_lock, flags);
+	list_splice_tail_init(&items, &prefetch_list);
+	queue_delayed_work(prefetch_wq, &prefetch_work,
+			   shrink ?  msecs_to_jiffies(SHRINK_DELAY) : 0);
+	spin_unlock_irqrestore(&work_lock, flags);
+
+	return 0;
+
+out_free:
+	list_for_each_entry_safe(info, tmp, &items, list) {
+		list_del(&info->list);
+		kfree(info);
+	}
+	return ret;
+}
+
+int qcom_secure_system_heap_prefetch(struct dma_buf_heap_prefetch_region *regions,
+				     size_t nr_regions)
+{
+	return __qcom_secure_system_heap_resize(regions, nr_regions, false);
+}
+EXPORT_SYMBOL(qcom_secure_system_heap_prefetch);
+
+int qcom_secure_system_heap_drain(struct dma_buf_heap_prefetch_region *regions,
+				  size_t nr_regions)
+{
+	return __qcom_secure_system_heap_resize(regions, nr_regions, true);
+}
+EXPORT_SYMBOL(qcom_secure_system_heap_drain);
+
+static enum dynamic_pool_callback_ret free_secure_pages(struct dynamic_page_pool *pool,
+							struct list_head *pages,
+							int num_pages)
+{
+	struct page *page;
+	struct sg_table sgt;
+	struct scatterlist *sg;
+	int ret;
+
+	if (!num_pages)
+		return DYNAMIC_POOL_SUCCESS;
+
+	ret = sg_alloc_table(&sgt, num_pages, GFP_KERNEL);
+	if (ret)
+		return DYNAMIC_POOL_FAILURE;
+
+	sg = sgt.sgl;
+	list_for_each_entry(page, pages, lru) {
+		sg_set_page(sg, page, page_size(page), 0);
+		sg = sg_next(sg);
+	}
+
+	ret = hyp_unassign_sg_from_flags(&sgt, pool->vmid, true);
+	sg_free_table(&sgt);
+
+	if (ret)
+		return DYNAMIC_POOL_FAILURE;
+
+	return DYNAMIC_POOL_SUCCESS;
+}
+
+static void page_list_merge(struct list_head *secure_pages,
+			    struct list_head *non_secure_pages,
+			    struct scatterlist *sg,
+			    struct scatterlist *non_secure_sg)
+{
+	bool is_secure_head, is_non_secure_head;
+	struct page *secure_page = list_first_entry(secure_pages, struct page, lru);
+	struct page *non_secure_page = list_first_entry(non_secure_pages, struct page, lru);
+
+	do {
+		is_secure_head = list_entry_is_head(secure_page, secure_pages, lru);
+		is_non_secure_head = list_entry_is_head(non_secure_page, non_secure_pages, lru);
+
+		if (!is_secure_head && !is_non_secure_head) {
+			if (page_size(non_secure_page) >= page_size(secure_page)) {
+				sg_set_page(sg, non_secure_page,
+					    page_size(non_secure_page), 0);
+				sg_set_page(non_secure_sg, non_secure_page,
+					    page_size(non_secure_page), 0);
+
+				non_secure_page = list_next_entry(non_secure_page, lru);
+				non_secure_sg = sg_next(non_secure_sg);
+			} else {
+				sg_set_page(sg, secure_page,
+					    page_size(secure_page), 0);
+				secure_page = list_next_entry(secure_page, lru);
+			}
+		} else if (!is_non_secure_head) {
+			sg_set_page(sg, non_secure_page,
+					page_size(non_secure_page), 0);
+			sg_set_page(non_secure_sg, non_secure_page,
+					page_size(non_secure_page), 0);
+
+			non_secure_page = list_next_entry(non_secure_page, lru);
+			non_secure_sg = sg_next(non_secure_sg);
+		} else if (!is_secure_head) {
+			sg_set_page(sg, secure_page,
+					page_size(secure_page), 0);
+			secure_page = list_next_entry(secure_page, lru);
+		}
+
+		sg = sg_next(sg);
+	} while (sg);
+}
+
+static int page_to_pool_ind(struct page *page)
+{
+	int i;
+
+	for (i = 0; i < NUM_ORDERS; i++)
+		if (compound_order(page) == orders[i])
+			break;
+	return i;
+}
+
+static void system_heap_free(struct qcom_sg_buffer *buffer)
+{
+	struct qcom_secure_system_heap *sys_heap;
+	struct sg_table *table;
+	struct scatterlist *sg;
+	int i, j;
+
+	sys_heap = dma_heap_get_drvdata(buffer->heap);
+	table = &buffer->sg_table;
+
+	for_each_sg(table->sgl, sg, table->nents, i) {
+		struct page *page = sg_page(sg);
+
+		for (j = 0; j < NUM_ORDERS; j++) {
+			if (compound_order(page) == orders[j])
+				break;
+		}
+		dynamic_page_pool_free(sys_heap->pool_list[j], page);
+	}
+	sg_free_table(table);
+	kfree(buffer);
+}
+
+static struct page *alloc_largest_available(struct dynamic_page_pool **pools,
+					    unsigned long size,
+					    unsigned int max_order,
+					    bool *page_from_secure_pool)
+{
+	struct page *page = NULL;
+	int i;
+	struct dynamic_page_pool **qcom_sys_heap_pools = get_sys_heap_page_pool();
+
+	if (!qcom_sys_heap_pools) {
+		pr_err("%s: Couldn't obtain the pools for the system heap!\n", __func__);
+		return NULL;
+	}
+
+	*page_from_secure_pool = true;
+
+	for (i = 0; i < NUM_ORDERS; i++) {
+		if (size <  (PAGE_SIZE << orders[i]))
+			continue;
+		if (max_order < orders[i])
+			continue;
+
+		mutex_lock(&pools[i]->mutex);
+		if (pools[i]->high_count)
+			page = dynamic_page_pool_remove(pools[i], true);
+		else if (pools[i]->low_count)
+			page = dynamic_page_pool_remove(pools[i], false);
+		mutex_unlock(&pools[i]->mutex);
+
+		if (!page)
+			continue;
+		return page;
+	}
+
+	*page_from_secure_pool = false;
+
+	return qcom_sys_heap_alloc_largest_available(qcom_sys_heap_pools,
+						     size,
+						     max_order);
+}
+
+static struct dma_buf *system_heap_allocate(struct dma_heap *heap,
+					       unsigned long len,
+					       unsigned long fd_flags,
+					       unsigned long heap_flags)
+{
+	struct qcom_secure_system_heap *sys_heap;
+	struct qcom_sg_buffer *buffer;
+	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
+	unsigned long size_remaining = len;
+	unsigned int max_order = orders[0];
+	bool page_from_secure_pool;
+	struct dma_buf *dmabuf;
+	struct sg_table *table, non_secure_table;
+	struct scatterlist *sg, *non_secure_sg = NULL;
+	LIST_HEAD(secure_pages);
+	LIST_HEAD(non_secure_pages);
+	struct page *page, *tmp_page;
+	int total_pages = 0, num_non_secure_pages = 0;
+	int ret = -ENOMEM, hyp_ret = 0;
+	int perms;
+	int vmid;
+
+	buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
+	if (!buffer)
+		return ERR_PTR(-ENOMEM);
+
+	sys_heap = dma_heap_get_drvdata(heap);
+
+	INIT_LIST_HEAD(&buffer->attachments);
+	mutex_init(&buffer->lock);
+	buffer->heap = heap;
+	buffer->len = len;
+	buffer->uncached = true;
+	buffer->free = system_heap_free;
+
+	while (size_remaining > 0) {
+		/*
+		 * Avoid trying to allocate memory if the process
+		 * has been killed by SIGKILL
+		 */
+		if (fatal_signal_pending(current))
+			goto free_buffer;
+
+		page = alloc_largest_available(sys_heap->pool_list,
+					       size_remaining,
+					       max_order,
+					       &page_from_secure_pool);
+		if (!page)
+			goto free_buffer;
+
+		if (page_from_secure_pool) {
+			list_add_tail(&page->lru, &secure_pages);
+		} else {
+			num_non_secure_pages++;
+			list_add_tail(&page->lru, &non_secure_pages);
+		}
+
+		size_remaining -= page_size(page);
+		max_order = compound_order(page);
+		total_pages++;
+	}
+
+	table = &buffer->sg_table;
+	if (sg_alloc_table(table, total_pages, GFP_KERNEL))
+		goto free_buffer;
+	sg = table->sgl;
+
+	if (num_non_secure_pages) {
+		if (sg_alloc_table(&non_secure_table, num_non_secure_pages, GFP_KERNEL))
+			goto free_sg;
+		non_secure_sg = non_secure_table.sgl;
+	}
+
+	page_list_merge(&secure_pages, &non_secure_pages, sg, non_secure_sg);
+
+	if (num_non_secure_pages) {
+		dma_map_sgtable(dma_heap_get_dev(heap), &non_secure_table, DMA_BIDIRECTIONAL, 0);
+		dma_unmap_sgtable(dma_heap_get_dev(heap), &non_secure_table, DMA_BIDIRECTIONAL, 0);
+
+		hyp_ret = hyp_assign_sg_from_flags(&non_secure_table, sys_heap->vmid, true);
+		if (hyp_ret)
+			goto free_non_secure_sg;
+	}
+
+	perms = msm_secure_get_vmid_perms(sys_heap->vmid);
+	vmid = get_secure_vmid(sys_heap->vmid);
+	buffer->vmperm = mem_buf_vmperm_alloc_staticvm(table,
+				&vmid, &perms, 1);
+
+	if (IS_ERR(buffer->vmperm)) {
+		ret = PTR_ERR(buffer->vmperm);
+		goto hyp_unassign;
+	}
+
+	/* create the dmabuf */
+	exp_info.exp_name = dma_heap_get_name(heap);
+	exp_info.ops = &qcom_sg_buf_ops.dma_ops;
+	exp_info.size = buffer->len;
+	exp_info.flags = fd_flags;
+	exp_info.priv = buffer;
+	dmabuf = mem_buf_dma_buf_export(&exp_info, &qcom_sg_buf_ops);
+	if (IS_ERR(dmabuf)) {
+		ret = PTR_ERR(dmabuf);
+		goto vmperm_release;
+	}
+
+	if (num_non_secure_pages)
+		sg_free_table(&non_secure_table);
+
+	return dmabuf;
+
+vmperm_release:
+	mem_buf_vmperm_release(buffer->vmperm);
+
+hyp_unassign:
+	/* We check PagePrivate() below to see if we've reclaimed a particular page */
+	hyp_ret = hyp_unassign_sg_from_flags(&non_secure_table, sys_heap->vmid, true);
+
+free_non_secure_sg:
+	if (num_non_secure_pages)
+		sg_free_table(&non_secure_table);
+free_sg:
+	sg_free_table(table);
+free_buffer:
+	list_for_each_entry_safe(page, tmp_page, &secure_pages, lru)
+		dynamic_page_pool_free(sys_heap->pool_list[page_to_pool_ind(page)], page);
+	if (!hyp_ret)
+		list_for_each_entry_safe(page, tmp_page, &non_secure_pages, lru)
+			__free_pages(page, compound_order(page));
+
+	kfree(buffer);
+
+	return ERR_PTR(ret);
+}
+
+
+static const struct dma_heap_ops system_heap_ops = {
+	.allocate = system_heap_allocate,
+};
+
+static int create_prefetch_workqueue(void)
+{
+	static bool registered;
+
+	if (registered)
+		return 0;
+
+	INIT_DELAYED_WORK(&prefetch_work,
+			  secure_system_heap_prefetch_work);
+
+	prefetch_wq = alloc_workqueue("system_secure_prefetch_wq",
+				      WQ_UNBOUND | WQ_FREEZABLE, 0);
+	if (!prefetch_wq) {
+		pr_err("Failed to create system secure prefetch workqueue\n");
+		return -ENOMEM;
+	}
+
+	registered = true;
+	return 0;
+}
+
+void qcom_secure_system_heap_create(const char *name, const char *secure_system_alias, int vmid)
+{
+	struct dma_heap_export_info exp_info;
+	struct dma_heap *heap;
+	struct qcom_secure_system_heap *sys_heap;
+	int ret;
+
+	if (get_secure_vmid(vmid) == -EINVAL || hweight_long(vmid) != 1) {
+		pr_err("Invalid VMID or supplied more than one VMID\n");
+		ret = -EINVAL;
+		goto out;
+	}
+
+	ret = dynamic_page_pool_init_shrinker();
+	if (ret)
+		goto out;
+
+	ret = create_prefetch_workqueue();
+	if (ret)
+		goto out;
+
+	sys_heap = kzalloc(sizeof(*sys_heap), GFP_KERNEL);
+	if (!sys_heap) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	sys_heap->vmid = vmid;
+	sys_heap->pool_list = dynamic_page_pool_create_pools(vmid, free_secure_pages);
+	if (IS_ERR(sys_heap->pool_list)) {
+		ret = PTR_ERR(sys_heap->pool_list);
+		goto free_heap;
+	}
+
+	exp_info.name = name;
+	exp_info.ops = &system_heap_ops;
+	exp_info.priv = sys_heap;
+
+	heap = dma_heap_add(&exp_info);
+	if (IS_ERR(heap)) {
+		ret = PTR_ERR(heap);
+		goto free_pools;
+	}
+
+	dma_coerce_mask_and_coherent(dma_heap_get_dev(heap), DMA_BIT_MASK(64));
+
+	list_add(&sys_heap->list, &secure_heaps);
+
+	pr_info("%s: DMA-BUF Heap: Created '%s'\n", __func__, name);
+
+	if (secure_system_alias != NULL) {
+		exp_info.name = secure_system_alias;
+
+		heap = dma_heap_add(&exp_info);
+		if (IS_ERR(heap)) {
+			pr_err("%s: Failed to create '%s', error is %d\n", __func__,
+			       secure_system_alias, PTR_ERR(heap));
+			return;
+		}
+
+		dma_coerce_mask_and_coherent(dma_heap_get_dev(heap), DMA_BIT_MASK(64));
+
+		pr_info("%s: DMA-BUF Heap: Created '%s'\n", __func__, secure_system_alias);
+	}
+
+	return;
+
+free_pools:
+	dynamic_page_pool_release_pools(sys_heap->pool_list);
+
+free_heap:
+	kfree(sys_heap);
+
+out:
+	pr_err("%s: Failed to create '%s', error is %d\n", __func__, name, ret);
+}

drivers/dma-buf/heaps/qcom_secure_system_heap.h

@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
+ */
+
+#ifndef _QCOM_SECURE_SYSTEM_HEAP_H
+#define _QCOM_SECURE_SYSTEM_HEAP_H
+
+#include <linux/dma-heap.h>
+#include <linux/err.h>
+#include "qcom_dynamic_page_pool.h"
+
+struct qcom_secure_system_heap {
+	struct dynamic_page_pool **pool_list;
+	int vmid;
+	struct list_head list;
+};
+
+#ifdef CONFIG_QCOM_DMABUF_HEAPS_SYSTEM_SECURE
+void qcom_secure_system_heap_create(const char *name, const char *secure_system_alias,
+				    int vmid);
+#else
+static void qcom_secure_system_heap_create(const char *name, const char *secure_system_alias,
+					   int vmid)
+{
+
+}
+#endif
+
+#endif /* _QCOM_SECURE_SYSTEM_HEAP_H */

drivers/dma-buf/heaps/qcom_sg_ops.c

@@ -0,0 +1,544 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * mmap() algorithm taken from drivers/staging/android/ion/ion_heap.c as
+ * of commit a3ec289e74b4 ("arm-smmu: Fix missing qsmmuv500 callback")
+ *
+ * Copyright (C) 2011 Google, Inc.
+ * Copyright (C) 2019, 2020 Linaro Ltd.
+ *
+ * Portions based off of Andrew Davis' SRAM heap:
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ *	Andrew F. Davis <afd@ti.com>
+ *
+ * These ops were base of the ops in drivers/dma-buf/heaps/system-heap.c from
+ * https://lore.kernel.org/lkml/20201017013255.43568-2-john.stultz@linaro.org/
+ *
+ * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
+ */
+
+#include <linux/dma-buf.h>
+#include <linux/dma-mapping.h>
+#include <linux/dma-heap.h>
+#include <linux/err.h>
+#include <linux/highmem.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/qcom_dma_heap.h>
+#include <linux/msm_dma_iommu_mapping.h>
+
+#include "qcom_sg_ops.h"
+
+static struct sg_table *dup_sg_table(struct sg_table *table)
+{
+	struct sg_table *new_table;
+	int ret, i;
+	struct scatterlist *sg, *new_sg;
+
+	new_table = kzalloc(sizeof(*new_table), GFP_KERNEL);
+	if (!new_table)
+		return ERR_PTR(-ENOMEM);
+
+	ret = sg_alloc_table(new_table, table->orig_nents, GFP_KERNEL);
+	if (ret) {
+		kfree(new_table);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	new_sg = new_table->sgl;
+	for_each_sgtable_sg(table, sg, i) {
+		sg_set_page(new_sg, sg_page(sg), sg->length, sg->offset);
+		new_sg = sg_next(new_sg);
+	}
+
+	return new_table;
+}
+
+static int qcom_sg_attach(struct dma_buf *dmabuf,
+			  struct dma_buf_attachment *attachment)
+{
+	struct qcom_sg_buffer *buffer = dmabuf->priv;
+	struct dma_heap_attachment *a;
+	struct sg_table *table;
+
+	a = kzalloc(sizeof(*a), GFP_KERNEL);
+	if (!a)
+		return -ENOMEM;
+
+	table = dup_sg_table(&buffer->sg_table);
+	if (IS_ERR(table)) {
+		kfree(a);
+		return -ENOMEM;
+	}
+
+	a->table = table;
+	a->dev = attachment->dev;
+	INIT_LIST_HEAD(&a->list);
+	a->mapped = false;
+
+	attachment->priv = a;
+
+	mutex_lock(&buffer->lock);
+	list_add(&a->list, &buffer->attachments);
+	mutex_unlock(&buffer->lock);
+
+	return 0;
+}
+
+static void qcom_sg_detach(struct dma_buf *dmabuf,
+			   struct dma_buf_attachment *attachment)
+{
+	struct qcom_sg_buffer *buffer = dmabuf->priv;
+	struct dma_heap_attachment *a = attachment->priv;
+
+	mutex_lock(&buffer->lock);
+	list_del(&a->list);
+	mutex_unlock(&buffer->lock);
+
+	sg_free_table(a->table);
+	kfree(a->table);
+	kfree(a);
+}
+
+static struct sg_table *qcom_sg_map_dma_buf(struct dma_buf_attachment *attachment,
+					    enum dma_data_direction direction)
+{
+	struct dma_heap_attachment *a = attachment->priv;
+	struct sg_table *table = a->table;
+	struct qcom_sg_buffer *buffer;
+	struct mem_buf_vmperm *vmperm;
+	unsigned long attrs = attachment->dma_map_attrs;
+	int ret;
+
+	buffer = attachment->dmabuf->priv;
+	vmperm = buffer->vmperm;
+
+	/* Prevent map/unmap during begin/end_cpu_access */
+	mutex_lock(&buffer->lock);
+
+	/* Ensure VM permissions are constant while the buffer is mapped */
+	mem_buf_vmperm_pin(vmperm);
+	if (buffer->uncached || !mem_buf_vmperm_can_cmo(vmperm))
+		attrs |= DMA_ATTR_SKIP_CPU_SYNC;
+
+	if (attrs & DMA_ATTR_DELAYED_UNMAP)
+		ret = msm_dma_map_sgtable(attachment->dev, table, direction,
+					  attachment->dmabuf, attrs);
+	else
+		ret = dma_map_sgtable(attachment->dev, table, direction, attrs);
+
+	if (ret) {
+		table = ERR_PTR(-ENOMEM);
+		goto err_map_sgtable;
+	}
+
+	a->mapped = true;
+	mutex_unlock(&buffer->lock);
+	return table;
+
+err_map_sgtable:
+	mem_buf_vmperm_unpin(vmperm);
+	mutex_unlock(&buffer->lock);
+	return table;
+}
+
+static void qcom_sg_unmap_dma_buf(struct dma_buf_attachment *attachment,
+				  struct sg_table *table,
+				  enum dma_data_direction direction)
+{
+	struct dma_heap_attachment *a = attachment->priv;
+	struct qcom_sg_buffer *buffer;
+	struct mem_buf_vmperm *vmperm;
+	unsigned long attrs = attachment->dma_map_attrs;
+
+	buffer = attachment->dmabuf->priv;
+	vmperm = buffer->vmperm;
+
+	/* Prevent map/unmap during begin/end_cpu_access */
+	mutex_lock(&buffer->lock);
+
+	if (buffer->uncached || !mem_buf_vmperm_can_cmo(vmperm))
+		attrs |= DMA_ATTR_SKIP_CPU_SYNC;
+
+	a->mapped = false;
+	if (attrs & DMA_ATTR_DELAYED_UNMAP)
+		msm_dma_unmap_sgtable(attachment->dev, table, direction,
+				      attachment->dmabuf, attrs);
+	else
+		dma_unmap_sgtable(attachment->dev, table, direction, attrs);
+	mem_buf_vmperm_unpin(vmperm);
+	mutex_unlock(&buffer->lock);
+}
+
+static int qcom_sg_dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
+					    enum dma_data_direction direction)
+{
+	struct qcom_sg_buffer *buffer = dmabuf->priv;
+	struct dma_heap_attachment *a;
+
+	if (buffer->uncached)
+		return 0;
+
+	mutex_lock(&buffer->lock);
+
+	/* Keep the same behavior as ion by returning 0 instead of -EPERM */
+	if (!mem_buf_vmperm_can_cmo(buffer->vmperm)) {
+		mutex_unlock(&buffer->lock);
+		return 0;
+	}
+
+	if (buffer->vmap_cnt)
+		invalidate_kernel_vmap_range(buffer->vaddr, buffer->len);
+
+	list_for_each_entry(a, &buffer->attachments, list) {
+		if (!a->mapped)
+			continue;
+		dma_sync_sgtable_for_cpu(a->dev, a->table, direction);
+	}
+
+	mutex_unlock(&buffer->lock);
+
+	return 0;
+}
+
+static int qcom_sg_dma_buf_end_cpu_access(struct dma_buf *dmabuf,
+					  enum dma_data_direction direction)
+{
+	struct qcom_sg_buffer *buffer = dmabuf->priv;
+	struct dma_heap_attachment *a;
+
+	if (buffer->uncached)
+		return 0;
+
+	mutex_lock(&buffer->lock);
+
+	/* Keep the same behavior as ion by returning 0 instead of -EPERM */
+	if (!mem_buf_vmperm_can_cmo(buffer->vmperm)) {
+		mutex_unlock(&buffer->lock);
+		return 0;
+	}
+
+	if (buffer->vmap_cnt)
+		flush_kernel_vmap_range(buffer->vaddr, buffer->len);
+
+	list_for_each_entry(a, &buffer->attachments, list) {
+		if (!a->mapped)
+			continue;
+		dma_sync_sgtable_for_device(a->dev, a->table, direction);
+	}
+	mutex_unlock(&buffer->lock);
+
+	return 0;
+}
+
+static int sgl_sync_range(struct device *dev, struct scatterlist *sgl,
+			  unsigned int nents, unsigned long offset,
+			  unsigned long length,
+			  enum dma_data_direction dir, bool for_cpu)
+{
+	int i;
+	struct scatterlist *sg;
+	unsigned int len = 0;
+	dma_addr_t sg_dma_addr;
+
+	for_each_sg(sgl, sg, nents, i) {
+		if (sg_dma_len(sg) == 0)
+			break;
+
+		if (i > 0) {
+			pr_warn_ratelimited("Partial cmo only supported with 1 segment\n"
+				"is dma_set_max_seg_size being set on dev:%s\n",
+				dev_name(dev));
+			return -EINVAL;
+		}
+	}
+
+	for_each_sg(sgl, sg, nents, i) {
+		unsigned int sg_offset, sg_left, size = 0;
+
+		if (i == 0)
+			sg_dma_addr = sg_dma_address(sg);
+
+		len += sg->length;
+		if (len <= offset) {
+			sg_dma_addr += sg->length;
+			continue;
+		}
+
+		sg_left = len - offset;
+		sg_offset = sg->length - sg_left;
+
+		size = (length < sg_left) ? length : sg_left;
+		if (for_cpu)
+			dma_sync_single_range_for_cpu(dev, sg_dma_addr,
+						      sg_offset, size, dir);
+		else
+			dma_sync_single_range_for_device(dev, sg_dma_addr,
+							 sg_offset, size, dir);
+
+		offset += size;
+		length -= size;
+		sg_dma_addr += sg->length;
+
+		if (length == 0)
+			break;
+	}
+
+	return 0;
+}
+
+static int qcom_sg_dma_buf_begin_cpu_access_partial(struct dma_buf *dmabuf,
+						    enum dma_data_direction dir,
+						    unsigned int offset,
+						    unsigned int len)
+{
+	struct qcom_sg_buffer *buffer = dmabuf->priv;
+	struct dma_heap_attachment *a;
+	int ret = 0;
+
+	if (buffer->uncached)
+		return 0;
+
+	mutex_lock(&buffer->lock);
+
+	/* Keep the same behavior as ion by returning 0 instead of -EPERM */
+	if (!mem_buf_vmperm_can_cmo(buffer->vmperm)) {
+		mutex_unlock(&buffer->lock);
+		return 0;
+	}
+
+	if (buffer->vmap_cnt)
+		invalidate_kernel_vmap_range(buffer->vaddr + offset, len);
+
+	list_for_each_entry(a, &buffer->attachments, list) {
+		if (!a->mapped)
+			continue;
+
+		ret = sgl_sync_range(a->dev, a->table->sgl, a->table->orig_nents,
+				     offset, len, dir, true);
+	}
+	mutex_unlock(&buffer->lock);
+
+	return ret;
+}
+
+static int qcom_sg_dma_buf_end_cpu_access_partial(struct dma_buf *dmabuf,
+					      enum dma_data_direction direction,
+					      unsigned int offset,
+					      unsigned int len)
+{
+	struct qcom_sg_buffer *buffer = dmabuf->priv;
+	struct dma_heap_attachment *a;
+	int ret = 0;
+
+	if (buffer->uncached)
+		return 0;
+
+	mutex_lock(&buffer->lock);
+
+	/* Keep the same behavior as ion by returning 0 instead of -EPERM */
+	if (!mem_buf_vmperm_can_cmo(buffer->vmperm)) {
+		mutex_unlock(&buffer->lock);
+		return 0;
+	}
+
+	if (buffer->vmap_cnt)
+		flush_kernel_vmap_range(buffer->vaddr + offset, len);
+
+	list_for_each_entry(a, &buffer->attachments, list) {
+		if (!a->mapped)
+			continue;
+
+		ret = sgl_sync_range(a->dev, a->table->sgl, a->table->orig_nents,
+				     offset, len, direction, false);
+	}
+	mutex_unlock(&buffer->lock);
+
+	return ret;
+}
+
+static void qcom_sg_vm_ops_open(struct vm_area_struct *vma)
+{
+	struct mem_buf_vmperm *vmperm = vma->vm_private_data;
+
+	mem_buf_vmperm_pin(vmperm);
+}
+
+static void qcom_sg_vm_ops_close(struct vm_area_struct *vma)
+{
+	struct mem_buf_vmperm *vmperm = vma->vm_private_data;
+
+	mem_buf_vmperm_unpin(vmperm);
+}
+
+static const struct vm_operations_struct qcom_sg_vm_ops = {
+	.open = qcom_sg_vm_ops_open,
+	.close = qcom_sg_vm_ops_close,
+};
+
+static int qcom_sg_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
+{
+	struct qcom_sg_buffer *buffer = dmabuf->priv;
+	struct sg_table *table = &buffer->sg_table;
+	struct scatterlist *sg;
+	unsigned long addr = vma->vm_start;
+	unsigned long offset = vma->vm_pgoff * PAGE_SIZE;
+	int ret;
+	int i;
+
+	mem_buf_vmperm_pin(buffer->vmperm);
+	if (!mem_buf_vmperm_can_mmap(buffer->vmperm, vma)) {
+		mem_buf_vmperm_unpin(buffer->vmperm);
+		return -EPERM;
+	}
+
+	vma->vm_ops = &qcom_sg_vm_ops;
+	vma->vm_private_data = buffer->vmperm;
+	if (buffer->uncached)
+		vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
+
+	for_each_sg(table->sgl, sg, table->nents, i) {
+		struct page *page = sg_page(sg);
+		unsigned long remainder = vma->vm_end - addr;
+		unsigned long len = sg->length;
+
+		if (offset >= sg->length) {
+			offset -= sg->length;
+			continue;
+		} else if (offset) {
+			page += offset / PAGE_SIZE;
+			len = sg->length - offset;
+			offset = 0;
+		}
+		len = min(len, remainder);
+		ret = remap_pfn_range(vma, addr, page_to_pfn(page), len,
+				      vma->vm_page_prot);
+		if (ret) {
+			mem_buf_vmperm_unpin(buffer->vmperm);
+			return ret;
+		}
+		addr += len;
+		if (addr >= vma->vm_end)
+			return 0;
+	}
+	return 0;
+}
+
+static void *qcom_sg_do_vmap(struct qcom_sg_buffer *buffer)
+{
+	struct sg_table *table = &buffer->sg_table;
+	int npages = PAGE_ALIGN(buffer->len) / PAGE_SIZE;
+	struct page **pages = vmalloc(sizeof(struct page *) * npages);
+	struct page **tmp = pages;
+	struct sg_page_iter piter;
+	pgprot_t pgprot = PAGE_KERNEL;
+	void *vaddr;
+
+	if (!pages)
+		return ERR_PTR(-ENOMEM);
+
+	if (buffer->uncached)
+		pgprot = pgprot_writecombine(PAGE_KERNEL);
+
+	for_each_sgtable_page(table, &piter, 0) {
+		WARN_ON(tmp - pages >= npages);
+		*tmp++ = sg_page_iter_page(&piter);
+	}
+
+	vaddr = vmap(pages, npages, VM_MAP, pgprot);
+	vfree(pages);
+
+	if (!vaddr)
+		return ERR_PTR(-ENOMEM);
+
+	return vaddr;
+}
+
+static int qcom_sg_vmap(struct dma_buf *dmabuf, struct dma_buf_map *map)
+{
+	struct qcom_sg_buffer *buffer = dmabuf->priv;
+	void *vaddr;
+	int ret = 0;
+
+	mem_buf_vmperm_pin(buffer->vmperm);
+	if (!mem_buf_vmperm_can_vmap(buffer->vmperm)) {
+		mem_buf_vmperm_unpin(buffer->vmperm);
+		return -EPERM;
+	}
+
+	mutex_lock(&buffer->lock);
+	if (buffer->vmap_cnt) {
+		buffer->vmap_cnt++;
+		dma_buf_map_set_vaddr(map, buffer->vaddr);
+		goto out;
+	}
+
+	vaddr = qcom_sg_do_vmap(buffer);
+	if (IS_ERR(vaddr)) {
+		ret = PTR_ERR(vaddr);
+		mem_buf_vmperm_unpin(buffer->vmperm);
+		goto out;
+	}
+
+	buffer->vaddr = vaddr;
+	buffer->vmap_cnt++;
+	dma_buf_map_set_vaddr(map, buffer->vaddr);
+out:
+	mutex_unlock(&buffer->lock);
+
+	return ret;
+}
+
+static void qcom_sg_vunmap(struct dma_buf *dmabuf, struct dma_buf_map *map)
+{
+	struct qcom_sg_buffer *buffer = dmabuf->priv;
+
+	mutex_lock(&buffer->lock);
+	if (!--buffer->vmap_cnt) {
+		vunmap(buffer->vaddr);
+		buffer->vaddr = NULL;
+	}
+	mem_buf_vmperm_unpin(buffer->vmperm);
+	mutex_unlock(&buffer->lock);
+	dma_buf_map_clear(map);
+}
+
+static void qcom_sg_release(struct dma_buf *dmabuf)
+{
+	struct qcom_sg_buffer *buffer = dmabuf->priv;
+
+	if (mem_buf_vmperm_release(buffer->vmperm))
+		return;
+
+	msm_dma_buf_freed(buffer);
+	buffer->free(buffer);
+}
+
+static struct mem_buf_vmperm *qcom_sg_lookup_vmperm(struct dma_buf *dmabuf)
+{
+	struct qcom_sg_buffer *buffer = dmabuf->priv;
+
+	return buffer->vmperm;
+}
+
+struct mem_buf_dma_buf_ops qcom_sg_buf_ops = {
+	.attach = qcom_sg_attach,
+	.lookup = qcom_sg_lookup_vmperm,
+	.dma_ops = {
+		.attach = NULL, /* Will be set by mem_buf_dma_buf_export */
+		.detach = qcom_sg_detach,
+		.map_dma_buf = qcom_sg_map_dma_buf,
+		.unmap_dma_buf = qcom_sg_unmap_dma_buf,
+		.begin_cpu_access = qcom_sg_dma_buf_begin_cpu_access,
+		.end_cpu_access = qcom_sg_dma_buf_end_cpu_access,
+		.begin_cpu_access_partial = qcom_sg_dma_buf_begin_cpu_access_partial,
+		.end_cpu_access_partial = qcom_sg_dma_buf_end_cpu_access_partial,
+		.mmap = qcom_sg_mmap,
+		.vmap = qcom_sg_vmap,
+		.vunmap = qcom_sg_vunmap,
+		.release = qcom_sg_release,
+	}
+};
+EXPORT_SYMBOL(qcom_sg_buf_ops);

drivers/dma-buf/heaps/qcom_sg_ops.h

@@ -0,0 +1,43 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2011 Google, Inc.
+ * Copyright (C) 2019, 2020 Linaro Ltd.
+ *
+ * Portions based off of Andrew Davis' SRAM heap:
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ *	Andrew F. Davis <afd@ti.com>
+ *
+ * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
+ */
+
+#ifndef _QCOM_SG_OPS_H
+#define _QCOM_SG_OPS_H
+
+#include <linux/scatterlist.h>
+#include <linux/dma-heap.h>
+#include <linux/device.h>
+#include <linux/mem-buf-exporter.h>
+
+struct qcom_sg_buffer {
+	struct dma_heap *heap;
+	struct list_head attachments;
+	struct mutex lock;
+	unsigned long len;
+	struct sg_table sg_table;
+	int vmap_cnt;
+	void *vaddr;
+	bool uncached;
+	struct mem_buf_vmperm *vmperm;
+	void (*free)(struct qcom_sg_buffer *buffer);
+};
+
+struct dma_heap_attachment {
+	struct device *dev;
+	struct sg_table *table;
+	struct list_head list;
+	bool mapped;
+};
+
+extern struct mem_buf_dma_buf_ops qcom_sg_buf_ops;
+
+#endif /* _QCOM_SG_OPS_H */

drivers/dma-buf/heaps/qcom_system_heap.c

@@ -0,0 +1,584 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * DMABUF System heap exporter
+ * Originally copied from: drivers/dma-buf/heaps/system_heap.c as of commit
+ * 263e38f82cbb ("dma-buf: heaps: Remove redundant heap identifier from system
+ * heap name")
+ *
+ * Additions taken from modifications to drivers/dma-buf/heaps/system-heap.c,
+ * from patches submitted, are listed below:
+ *
+ * Addition that modifies dma_buf ops to use SG tables taken from
+ * drivers/dma-buf/heaps/system-heap.c in:
+ * https://lore.kernel.org/lkml/20201017013255.43568-2-john.stultz@linaro.org/
+ *
+ * Addition that skips unneeded syncs in the dma_buf ops taken from
+ * https://lore.kernel.org/lkml/20201017013255.43568-5-john.stultz@linaro.org/
+ *
+ * Addition that tries to allocate higher order pages taken from
+ * https://lore.kernel.org/lkml/20201017013255.43568-6-john.stultz@linaro.org/
+ *
+ * Addition that implements an uncached heap taken from
+ * https://lore.kernel.org/lkml/20201017013255.43568-8-john.stultz@linaro.org/,
+ * with our own modificaitons made to account for core kernel changes that are
+ * a part of the patch series.
+ *
+ * Pooling functionality taken from:
+ * Git-repo: https://git.linaro.org/people/john.stultz/android-dev.git
+ * Branch: dma-buf-heap-perf
+ * Git-commit: 6f080eb67dce63c6efa57ef564ca4cd762ccebb0
+ * Git-commit: 6fb9593b928c4cb485bef4e88c59c6b9fdf11352
+ *
+ * Copyright (C) 2011 Google, Inc.
+ * Copyright (C) 2019, 2020 Linaro Ltd.
+ *
+ * Portions based off of Andrew Davis' SRAM heap:
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ *	Andrew F. Davis <afd@ti.com>
+ *
+ * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
+ */
+
+#include <linux/dma-buf.h>
+#include <linux/dma-mapping.h>
+#include <linux/dma-heap.h>
+#include <linux/err.h>
+#include <linux/highmem.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/kthread.h>
+#include <linux/qcom_dma_heap.h>
+#include <uapi/linux/sched/types.h>
+
+#include "qcom_dma_heap_secure_utils.h"
+#include "qcom_dynamic_page_pool.h"
+#include "qcom_sg_ops.h"
+#include "qcom_system_heap.h"
+
+#if IS_ENABLED(CONFIG_QCOM_DMABUF_HEAPS_PAGE_POOL_REFILL)
+#define DYNAMIC_POOL_FILL_MARK (100 * SZ_1M)
+#define DYNAMIC_POOL_LOW_MARK_PERCENT 40UL
+#define DYNAMIC_POOL_LOW_MARK ((DYNAMIC_POOL_FILL_MARK * DYNAMIC_POOL_LOW_MARK_PERCENT) / 100)
+
+#define DYNAMIC_POOL_REFILL_DEFER_WINDOW_MS 10
+#define DYNAMIC_POOL_KTHREAD_NICE_VAL 10
+
+static int get_dynamic_pool_fillmark(struct dynamic_page_pool *pool)
+{
+	return DYNAMIC_POOL_FILL_MARK / (PAGE_SIZE << pool->order);
+}
+
+static bool dynamic_pool_fillmark_reached(struct dynamic_page_pool *pool)
+{
+	return atomic_read(&pool->count) >= get_dynamic_pool_fillmark(pool);
+}
+
+static int get_dynamic_pool_lowmark(struct dynamic_page_pool *pool)
+{
+	return DYNAMIC_POOL_LOW_MARK / (PAGE_SIZE << pool->order);
+}
+
+static bool dynamic_pool_count_below_lowmark(struct dynamic_page_pool *pool)
+{
+	return atomic_read(&pool->count) < get_dynamic_pool_lowmark(pool);
+}
+
+/* Based on gfp_zone() in mm/mmzone.c since it is not exported. */
+static enum zone_type dynamic_pool_gfp_zone(gfp_t flags)
+{
+	enum zone_type z;
+	gfp_t local_flags = flags;
+	int bit;
+
+	bit = (__force int) ((local_flags) & GFP_ZONEMASK);
+
+	z = (GFP_ZONE_TABLE >> (bit * GFP_ZONES_SHIFT)) &
+					 ((1 << GFP_ZONES_SHIFT) - 1);
+	VM_BUG_ON((GFP_ZONE_BAD >> bit) & 1);
+	return z;
+}
+
+/*
+ * Based on __zone_watermark_ok() in mm/page_alloc.c since it is not exported.
+ *
+ * Return true if free base pages are above 'mark'. For high-order checks it
+ * will return true of the order-0 watermark is reached and there is at least
+ * one free page of a suitable size. Checking now avoids taking the zone lock
+ * to check in the allocation paths if no pages are free.
+ */
+static bool __dynamic_pool_zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
+					     int highest_zoneidx, long free_pages)
+{
+	long min = mark;
+	long unusable_free;
+	int o;
+
+	/*
+	 * Access to high atomic reserves is not required, and CMA should not be
+	 * used, since these allocations are non-movable.
+	 */
+	unusable_free = ((1 << order) - 1) + z->nr_reserved_highatomic;
+#ifdef CONFIG_CMA
+	unusable_free += zone_page_state(z, NR_FREE_CMA_PAGES);
+#endif
+
+	/* free_pages may go negative - that's OK */
+	free_pages -= unusable_free;
+
+	/*
+	 * Check watermarks for an order-0 allocation request. If these
+	 * are not met, then a high-order request also cannot go ahead
+	 * even if a suitable page happened to be free.
+	 *
+	 * 'min' can be taken as 'mark' since we do not expect these allocations
+	 * to require disruptive actions (such as running the OOM killer) or
+	 * a lot of effort.
+	 */
+	if (free_pages <= min + z->lowmem_reserve[highest_zoneidx])
+		return false;
+
+	/* If this is an order-0 request then the watermark is fine */
+	if (!order)
+		return true;
+
+	/* For a high-order request, check at least one suitable page is free */
+	for (o = order; o < MAX_ORDER; o++) {
+		struct free_area *area = &z->free_area[o];
+		int mt;
+
+		if (!area->nr_free)
+			continue;
+
+		for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) {
+			if (!free_area_empty(area, mt))
+				return true;
+		}
+	}
+
+	return false;
+}
+
+/* Based on zone_watermark_ok_safe from mm/page_alloc.c since it is not exported. */
+static bool dynamic_pool_zone_watermark_ok_safe(struct zone *z, unsigned int order,
+						unsigned long mark, int highest_zoneidx)
+{
+	long free_pages = zone_page_state(z, NR_FREE_PAGES);
+
+	if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
+		free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);
+
+	return __dynamic_pool_zone_watermark_ok(z, order, mark, highest_zoneidx, free_pages);
+}
+
+/* do a simple check to see if we are in any low memory situation */
+static bool dynamic_pool_refill_ok(struct dynamic_page_pool *pool)
+{
+	struct zone *zone;
+	int i, mark;
+	enum zone_type classzone_idx = dynamic_pool_gfp_zone(pool->gfp_mask);
+	s64 delta;
+
+	/* check if we are within the refill defer window */
+	delta = ktime_ms_delta(ktime_get(), pool->last_low_watermark_ktime);
+	if (delta < DYNAMIC_POOL_REFILL_DEFER_WINDOW_MS)
+		return false;
+
+	/*
+	 * make sure that if we allocate a pool->order page from buddy,
+	 * we don't put the zone watermarks below the high threshold.
+	 * This makes sure there's no unwanted repetitive refilling and
+	 * reclaiming of buddy pages on the pool.
+	 */
+	for (i = classzone_idx; i >= 0; i--) {
+		zone = &NODE_DATA(numa_node_id())->node_zones[i];
+
+		if (!strcmp(zone->name, "DMA32"))
+			continue;
+
+		mark = high_wmark_pages(zone);
+		mark += 1 << pool->order;
+		if (!dynamic_pool_zone_watermark_ok_safe(zone, pool->order, mark, classzone_idx)) {
+			pool->last_low_watermark_ktime = ktime_get();
+			return false;
+		}
+	}
+
+	return true;
+}
+
+static void dynamic_page_pool_refill(struct dynamic_page_pool *pool)
+{
+	struct page *page;
+	gfp_t gfp_refill = (pool->gfp_mask | __GFP_RECLAIM) & ~__GFP_NORETRY;
+
+	/* skip refilling order 0 pools */
+	if (!pool->order)
+		return;
+
+	while (!dynamic_pool_fillmark_reached(pool) && dynamic_pool_refill_ok(pool)) {
+		page = alloc_pages(gfp_refill, pool->order);
+		if (!page)
+			break;
+
+		dynamic_page_pool_add(pool, page);
+	}
+}
+
+static bool dynamic_pool_needs_refill(struct dynamic_page_pool *pool)
+{
+	return pool->order && dynamic_pool_count_below_lowmark(pool);
+}
+
+static int system_heap_refill_worker(void *data)
+{
+	struct dynamic_page_pool **pool_list = data;
+	int i;
+
+	for (;;) {
+		for (i = 0; i < NUM_ORDERS; i++) {
+			if (dynamic_pool_count_below_lowmark(pool_list[i]))
+				dynamic_page_pool_refill(pool_list[i]);
+		}
+
+		set_current_state(TASK_INTERRUPTIBLE);
+		if (unlikely(kthread_should_stop())) {
+			set_current_state(TASK_RUNNING);
+			break;
+		}
+		schedule();
+
+		set_current_state(TASK_RUNNING);
+	}
+
+	return 0;
+}
+
+static int system_heap_create_refill_worker(struct qcom_system_heap *sys_heap, const char *name)
+{
+	struct task_struct *refill_worker;
+	struct sched_attr attr = { .sched_nice = DYNAMIC_POOL_KTHREAD_NICE_VAL };
+	int ret;
+	int i;
+
+	refill_worker = kthread_run(system_heap_refill_worker, sys_heap->pool_list,
+				    "%s-pool-refill-thread", name);
+	if (IS_ERR(refill_worker)) {
+		pr_err("%s: failed to create %s-pool-refill-thread: %ld\n",
+			__func__, name, PTR_ERR(refill_worker));
+		return PTR_ERR(refill_worker);
+	}
+
+	ret = sched_setattr(refill_worker, &attr);
+	if (ret) {
+		pr_warn("%s: failed to set task priority for %s-pool-refill-thread: ret = %d\n",
+			__func__, name, ret);
+		kthread_stop(refill_worker);
+		return ret;
+	}
+
+	for (i = 0; i < NUM_ORDERS; i++)
+		sys_heap->pool_list[i]->refill_worker = refill_worker;
+
+	return ret;
+}
+
+static void system_heap_destroy_refill_worker(struct qcom_system_heap *sys_heap)
+{
+	kthread_stop(sys_heap->pool_list[0]->refill_worker);
+}
+#else
+static bool dynamic_pool_needs_refill(struct dynamic_page_pool *pool)
+{
+	return false;
+}
+
+static int system_heap_create_refill_worker(struct qcom_system_heap *sys_heap, const char *name)
+{
+	return 0;
+}
+
+static void system_heap_destroy_refill_worker(struct qcom_system_heap *sys_heap)
+{
+}
+#endif
+
+static int system_heap_clear_pages(struct page **pages, int num, pgprot_t pgprot)
+{
+	void *addr = vmap(pages, num, VM_MAP, pgprot);
+
+	if (!addr)
+		return -ENOMEM;
+	memset(addr, 0, PAGE_SIZE * num);
+	vunmap(addr);
+	return 0;
+}
+
+static int system_heap_zero_buffer(struct qcom_sg_buffer *buffer)
+{
+	struct sg_table *sgt = &buffer->sg_table;
+	struct sg_page_iter piter;
+	struct page *pages[32];
+	int p = 0;
+	int ret = 0;
+
+	for_each_sgtable_page(sgt, &piter, 0) {
+		pages[p++] = sg_page_iter_page(&piter);
+		if (p == ARRAY_SIZE(pages)) {
+			ret = system_heap_clear_pages(pages, p, PAGE_KERNEL);
+			if (ret)
+				return ret;
+			p = 0;
+		}
+	}
+	if (p)
+		ret = system_heap_clear_pages(pages, p, PAGE_KERNEL);
+
+	return ret;
+}
+
+static void system_heap_free(struct qcom_sg_buffer *buffer)
+{
+	struct qcom_system_heap *sys_heap;
+	struct sg_table *table;
+	struct scatterlist *sg;
+	int i, j;
+
+	sys_heap = dma_heap_get_drvdata(buffer->heap);
+	table = &buffer->sg_table;
+
+	/* Zero the buffer pages before adding back to the pool */
+	system_heap_zero_buffer(buffer);
+
+	for_each_sg(table->sgl, sg, table->nents, i) {
+		struct page *page = sg_page(sg);
+
+		for (j = 0; j < NUM_ORDERS; j++) {
+			if (compound_order(page) == orders[j])
+				break;
+		}
+		dynamic_page_pool_free(sys_heap->pool_list[j], page);
+	}
+	sg_free_table(table);
+	kfree(buffer);
+}
+
+struct page *qcom_sys_heap_alloc_largest_available(struct dynamic_page_pool **pools,
+						   unsigned long size,
+						   unsigned int max_order)
+{
+	struct page *page = NULL;
+	int i;
+
+	for (i = 0; i < NUM_ORDERS; i++) {
+		if (size <  (PAGE_SIZE << orders[i]))
+			continue;
+		if (max_order < orders[i])
+			continue;
+
+		mutex_lock(&pools[i]->mutex);
+		if (pools[i]->high_count)
+			page = dynamic_page_pool_remove(pools[i], true);
+		else if (pools[i]->low_count)
+			page = dynamic_page_pool_remove(pools[i], false);
+		mutex_unlock(&pools[i]->mutex);
+
+		if (!page)
+			page = alloc_pages(pools[i]->gfp_mask, pools[i]->order);
+		if (!page)
+			continue;
+
+		if (dynamic_pool_needs_refill(pools[i]))
+			wake_up_process(pools[i]->refill_worker);
+
+		return page;
+	}
+	return NULL;
+}
+
+static struct dma_buf *system_heap_allocate(struct dma_heap *heap,
+					       unsigned long len,
+					       unsigned long fd_flags,
+					       unsigned long heap_flags)
+{
+	struct qcom_system_heap *sys_heap;
+	struct qcom_sg_buffer *buffer;
+	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
+	unsigned long size_remaining = len;
+	unsigned int max_order = orders[0];
+	struct dma_buf *dmabuf;
+	struct sg_table *table;
+	struct scatterlist *sg;
+	struct list_head pages;
+	struct page *page, *tmp_page;
+	int i, ret = -ENOMEM;
+
+	buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
+	if (!buffer)
+		return ERR_PTR(-ENOMEM);
+
+	sys_heap = dma_heap_get_drvdata(heap);
+
+	INIT_LIST_HEAD(&buffer->attachments);
+	mutex_init(&buffer->lock);
+	buffer->heap = heap;
+	buffer->len = len;
+	buffer->uncached = sys_heap->uncached;
+	buffer->free = system_heap_free;
+
+	INIT_LIST_HEAD(&pages);
+	i = 0;
+	while (size_remaining > 0) {
+		/*
+		 * Avoid trying to allocate memory if the process
+		 * has been killed by SIGKILL
+		 */
+		if (fatal_signal_pending(current))
+			goto free_buffer;
+
+		page = qcom_sys_heap_alloc_largest_available(sys_heap->pool_list,
+							     size_remaining,
+							     max_order);
+		if (!page)
+			goto free_buffer;
+
+		list_add_tail(&page->lru, &pages);
+		size_remaining -= page_size(page);
+		max_order = compound_order(page);
+		i++;
+	}
+
+	table = &buffer->sg_table;
+	if (sg_alloc_table(table, i, GFP_KERNEL))
+		goto free_buffer;
+
+	sg = table->sgl;
+	list_for_each_entry_safe(page, tmp_page, &pages, lru) {
+		sg_set_page(sg, page, page_size(page), 0);
+		sg = sg_next(sg);
+		list_del(&page->lru);
+	}
+
+	/*
+	 * For uncached buffers, we need to initially flush cpu cache, since
+	 * the __GFP_ZERO on the allocation means the zeroing was done by the
+	 * cpu and thus it is likely cached. Map (and implicitly flush) and
+	 * unmap it now so we don't get corruption later on.
+	 */
+	if (buffer->uncached) {
+		dma_map_sgtable(dma_heap_get_dev(heap), table, DMA_BIDIRECTIONAL, 0);
+		dma_unmap_sgtable(dma_heap_get_dev(heap), table, DMA_BIDIRECTIONAL, 0);
+	}
+
+	buffer->vmperm = mem_buf_vmperm_alloc(table);
+
+	if (IS_ERR(buffer->vmperm)) {
+		ret = PTR_ERR(buffer->vmperm);
+		goto free_sg;
+	}
+
+	/* create the dmabuf */
+	exp_info.exp_name = dma_heap_get_name(heap);
+	exp_info.size = buffer->len;
+	exp_info.flags = fd_flags;
+	exp_info.priv = buffer;
+	dmabuf = mem_buf_dma_buf_export(&exp_info, &qcom_sg_buf_ops);
+	if (IS_ERR(dmabuf)) {
+		ret = PTR_ERR(dmabuf);
+		goto vmperm_release;
+	}
+
+	return dmabuf;
+
+vmperm_release:
+	mem_buf_vmperm_release(buffer->vmperm);
+free_sg:
+	sg_free_table(table);
+free_buffer:
+	list_for_each_entry_safe(page, tmp_page, &pages, lru)
+		__free_pages(page, compound_order(page));
+	kfree(buffer);
+
+	return ERR_PTR(ret);
+}
+
+static const struct dma_heap_ops system_heap_ops = {
+	.allocate = system_heap_allocate,
+};
+
+void qcom_system_heap_create(const char *name, const char *system_alias, bool uncached)
+{
+	struct dma_heap_export_info exp_info;
+	struct dma_heap *heap;
+	struct qcom_system_heap *sys_heap;
+	int ret;
+
+	ret = dynamic_page_pool_init_shrinker();
+	if (ret)
+		goto out;
+
+	sys_heap = kzalloc(sizeof(*sys_heap), GFP_KERNEL);
+	if (!sys_heap) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	exp_info.name = name;
+	exp_info.ops = &system_heap_ops;
+	exp_info.priv = sys_heap;
+
+	sys_heap->uncached = uncached;
+
+	sys_heap->pool_list = dynamic_page_pool_create_pools(0, NULL);
+	if (IS_ERR(sys_heap->pool_list)) {
+		ret = PTR_ERR(sys_heap->pool_list);
+		goto free_heap;
+	}
+
+	ret = system_heap_create_refill_worker(sys_heap, name);
+	if (ret)
+		goto free_pools;
+
+	heap = dma_heap_add(&exp_info);
+	if (IS_ERR(heap)) {
+		ret = PTR_ERR(heap);
+		goto stop_worker;
+	}
+
+	if (uncached)
+		dma_coerce_mask_and_coherent(dma_heap_get_dev(heap),
+					     DMA_BIT_MASK(64));
+
+	pr_info("%s: DMA-BUF Heap: Created '%s'\n", __func__, name);
+
+	if (system_alias != NULL) {
+		exp_info.name = system_alias;
+
+		heap = dma_heap_add(&exp_info);
+		if (IS_ERR(heap)) {
+			pr_err("%s: Failed to create '%s', error is %d\n", __func__,
+			       system_alias, PTR_ERR(heap));
+			return;
+		}
+
+		dma_coerce_mask_and_coherent(dma_heap_get_dev(heap), DMA_BIT_MASK(64));
+
+		pr_info("%s: DMA-BUF Heap: Created '%s'\n", __func__, system_alias);
+	}
+
+	return;
+
+stop_worker:
+	system_heap_destroy_refill_worker(sys_heap);
+
+free_pools:
+	dynamic_page_pool_release_pools(sys_heap->pool_list);
+
+free_heap:
+	kfree(sys_heap);
+
+out:
+	pr_err("%s: Failed to create '%s', error is %d\n", __func__, name, ret);
+}

drivers/dma-buf/heaps/qcom_system_heap.h

@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
+ */
+
+#ifndef _QCOM_SYSTEM_HEAP_H
+#define _QCOM_SYSTEM_HEAP_H
+
+#include <linux/dma-heap.h>
+#include <linux/err.h>
+#include "qcom_dynamic_page_pool.h"
+
+struct qcom_system_heap {
+	int uncached;
+	struct dynamic_page_pool **pool_list;
+};
+
+#ifdef CONFIG_QCOM_DMABUF_HEAPS_SYSTEM
+void qcom_system_heap_create(const char *name, const char *system_alias, bool uncached);
+struct page *qcom_sys_heap_alloc_largest_available(struct dynamic_page_pool **pools,
+						   unsigned long size,
+						   unsigned int max_order);
+#else
+struct page *qcom_sys_heap_alloc_largest_available(struct dynamic_page_pool **pools,
+						   unsigned long size,
+						   unsigned int max_order)
+{
+	return ERR_PTR(-EOPNOTSUPP);
+}
+
+static void qcom_system_heap_create(const char *name, const char *system_alias, bool uncached)
+{
+
+}
+#endif
+
+#endif /* _QCOM_SYSTEM_HEAP_H */

drivers/iommu/msm_dma_iommu_mapping.c

@@ -0,0 +1,480 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2015-2019, 2021, The Linux Foundation. All rights reserved.
+ */
+
+#include <linux/dma-map-ops.h>
+#include <linux/kernel.h>
+#include <linux/kref.h>
+#include <linux/slab.h>
+#include <linux/rbtree.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/err.h>
+#include <asm/barrier.h>
+
+#include <linux/msm_dma_iommu_mapping.h>
+#include <linux/qcom-dma-mapping.h>
+
+/**
+ * struct msm_iommu_map - represents a mapping of an ion buffer to an iommu
+ * @lnode - list node to exist in the buffer's list of iommu mappings
+ * @dev - Device this is mapped to. Used as key
+ * @sgl - The scatterlist for this mapping
+ * @nents - Number of entries in sgl
+ * @dir - The direction for the map.
+ * @meta - Backpointer to the meta this guy belongs to.
+ * @ref - for reference counting this mapping
+ * @attrs - dma mapping attributes
+ * @buf_start_addr - address of start of buffer
+ *
+ * Represents a mapping of one dma_buf buffer to a particular device
+ * and address range. There may exist other mappings of this buffer in
+ * different devices. All mappings will have the same cacheability and security.
+ */
+struct msm_iommu_map {
+	struct list_head lnode;
+	struct rb_node node;
+	struct device *dev;
+	struct scatterlist *sgl;
+	unsigned int nents;
+	enum dma_data_direction dir;
+	struct msm_iommu_meta *meta;
+	struct kref ref;
+	unsigned long attrs;
+	dma_addr_t buf_start_addr;
+};
+
+struct msm_iommu_meta {
+	struct rb_node node;
+	struct list_head iommu_maps;
+	struct kref ref;
+	struct mutex lock;
+	void *buffer;
+};
+
+static struct rb_root iommu_root;
+static DEFINE_MUTEX(msm_iommu_map_mutex);
+
+static void msm_iommu_meta_add(struct msm_iommu_meta *meta)
+{
+	struct rb_root *root = &iommu_root;
+	struct rb_node **p = &root->rb_node;
+	struct rb_node *parent = NULL;
+	struct msm_iommu_meta *entry;
+
+	while (*p) {
+		parent = *p;
+		entry = rb_entry(parent, struct msm_iommu_meta, node);
+
+		if (meta->buffer < entry->buffer)
+			p = &(*p)->rb_left;
+		else if (meta->buffer > entry->buffer)
+			p = &(*p)->rb_right;
+		else
+			pr_err("%s: dma_buf %pK already exists\n", __func__,
+			       entry->buffer);
+	}
+
+	rb_link_node(&meta->node, parent, p);
+	rb_insert_color(&meta->node, root);
+}
+
+static struct msm_iommu_meta *msm_iommu_meta_lookup(void *buffer)
+{
+	struct rb_root *root = &iommu_root;
+	struct rb_node **p = &root->rb_node;
+	struct rb_node *parent = NULL;
+	struct msm_iommu_meta *entry = NULL;
+
+	while (*p) {
+		parent = *p;
+		entry = rb_entry(parent, struct msm_iommu_meta, node);
+
+		if (buffer < entry->buffer)
+			p = &(*p)->rb_left;
+		else if (buffer > entry->buffer)
+			p = &(*p)->rb_right;
+		else
+			return entry;
+	}
+
+	return NULL;
+}
+
+static void msm_iommu_add(struct msm_iommu_meta *meta,
+			  struct msm_iommu_map *iommu)
+{
+	INIT_LIST_HEAD(&iommu->lnode);
+	list_add(&iommu->lnode, &meta->iommu_maps);
+}
+
+
+static struct msm_iommu_map *msm_iommu_lookup(struct msm_iommu_meta *meta,
+					      struct device *dev)
+{
+	struct msm_iommu_map *entry;
+
+	list_for_each_entry(entry, &meta->iommu_maps, lnode) {
+		if (entry->dev == dev)
+			return entry;
+	}
+
+	return NULL;
+}
+
+static struct msm_iommu_meta *msm_iommu_meta_create(struct dma_buf *dma_buf)
+{
+	struct msm_iommu_meta *meta;
+
+	meta = kzalloc(sizeof(*meta), GFP_KERNEL);
+
+	if (!meta)
+		return ERR_PTR(-ENOMEM);
+
+	INIT_LIST_HEAD(&meta->iommu_maps);
+	meta->buffer = dma_buf->priv;
+	kref_init(&meta->ref);
+	mutex_init(&meta->lock);
+	msm_iommu_meta_add(meta);
+
+	return meta;
+}
+
+static void msm_iommu_meta_put(struct msm_iommu_meta *meta);
+
+static struct scatterlist *clone_sgl(struct scatterlist *sg, int nents)
+{
+	struct scatterlist *next, *s;
+	int i;
+	struct sg_table table;
+
+	if (sg_alloc_table(&table, nents, GFP_KERNEL))
+		return NULL;
+	next = table.sgl;
+	for_each_sg(sg, s, nents, i) {
+		*next = *s;
+		next = sg_next(next);
+	}
+	return table.sgl;
+}
+
+static inline int __msm_dma_map_sg(struct device *dev, struct scatterlist *sg,
+				   int nents, enum dma_data_direction dir,
+				   struct dma_buf *dma_buf,
+				   unsigned long attrs)
+{
+	struct msm_iommu_map *iommu_map;
+	struct msm_iommu_meta *iommu_meta = NULL;
+	int ret = 0;
+	bool extra_meta_ref_taken = false;
+	int late_unmap = !(attrs & DMA_ATTR_NO_DELAYED_UNMAP);
+
+	mutex_lock(&msm_iommu_map_mutex);
+	iommu_meta = msm_iommu_meta_lookup(dma_buf->priv);
+
+	if (!iommu_meta) {
+		iommu_meta = msm_iommu_meta_create(dma_buf);
+
+		if (IS_ERR(iommu_meta)) {
+			mutex_unlock(&msm_iommu_map_mutex);
+			ret = PTR_ERR(iommu_meta);
+			goto out;
+		}
+		if (late_unmap) {
+			kref_get(&iommu_meta->ref);
+			extra_meta_ref_taken = true;
+		}
+	} else {
+		kref_get(&iommu_meta->ref);
+	}
+
+	mutex_unlock(&msm_iommu_map_mutex);
+
+	mutex_lock(&iommu_meta->lock);
+	iommu_map = msm_iommu_lookup(iommu_meta, dev);
+	if (!iommu_map) {
+		iommu_map = kmalloc(sizeof(*iommu_map), GFP_KERNEL);
+
+		if (!iommu_map) {
+			ret = -ENOMEM;
+			goto out_unlock;
+		}
+
+		ret = dma_map_sg_attrs(dev, sg, nents, dir, attrs);
+		if (!ret) {
+			kfree(iommu_map);
+			goto out_unlock;
+		}
+
+		iommu_map->sgl = clone_sgl(sg, nents);
+		if (!iommu_map->sgl) {
+			kfree(iommu_map);
+			ret = -ENOMEM;
+			goto out_unlock;
+		}
+		iommu_map->nents = nents;
+		iommu_map->dev = dev;
+		iommu_map->dir = dir;
+		iommu_map->attrs = attrs;
+		iommu_map->buf_start_addr = sg_phys(sg);
+
+		kref_init(&iommu_map->ref);
+		if (late_unmap)
+			kref_get(&iommu_map->ref);
+		iommu_map->meta = iommu_meta;
+		msm_iommu_add(iommu_meta, iommu_map);
+
+	} else {
+		if (nents == iommu_map->nents &&
+		    dir == iommu_map->dir &&
+		    (attrs & ~DMA_ATTR_SKIP_CPU_SYNC) ==
+		    (iommu_map->attrs & ~DMA_ATTR_SKIP_CPU_SYNC) &&
+		    sg_phys(sg) == iommu_map->buf_start_addr) {
+			struct scatterlist *sg_tmp = sg;
+			struct scatterlist *map_sg;
+			int i;
+
+			for_each_sg(iommu_map->sgl, map_sg, nents, i) {
+				sg_dma_address(sg_tmp) = sg_dma_address(map_sg);
+				sg_dma_len(sg_tmp) = sg_dma_len(map_sg);
+				if (sg_dma_len(map_sg) == 0)
+					break;
+
+				sg_tmp = sg_next(sg_tmp);
+				if (sg_tmp == NULL)
+					break;
+			}
+
+			kref_get(&iommu_map->ref);
+
+			if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
+				dma_sync_sg_for_device(dev, iommu_map->sgl,
+					iommu_map->nents, iommu_map->dir);
+
+			if (dev_is_dma_coherent(dev))
+				/*
+				 * Ensure all outstanding changes for coherent
+				 * buffers are applied to the cache before any
+				 * DMA occurs.
+				 */
+				dmb(ish);
+			ret = nents;
+		} else {
+			bool start_diff = (sg_phys(sg) !=
+					   iommu_map->buf_start_addr);
+
+			dev_err(dev, "lazy map request differs:\n"
+				"req dir:%d, original dir:%d\n"
+				"req nents:%d, original nents:%d\n"
+				"req map attrs:%lu, original map attrs:%lu\n"
+				"req buffer start address differs:%d\n",
+				dir, iommu_map->dir, nents,
+				iommu_map->nents, attrs, iommu_map->attrs,
+				start_diff);
+			ret = -EINVAL;
+		}
+	}
+	mutex_unlock(&iommu_meta->lock);
+	return ret;
+
+out_unlock:
+	mutex_unlock(&iommu_meta->lock);
+out:
+	if (!IS_ERR(iommu_meta)) {
+		if (extra_meta_ref_taken)
+			msm_iommu_meta_put(iommu_meta);
+		msm_iommu_meta_put(iommu_meta);
+	}
+	return ret;
+
+}
+
+/*
+ * We are not taking a reference to the dma_buf here. It is expected that
+ * clients hold reference to the dma_buf until they are done with mapping and
+ * unmapping.
+ */
+int msm_dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents,
+		   enum dma_data_direction dir, struct dma_buf *dma_buf,
+		   unsigned long attrs)
+{
+	int ret;
+
+	if (IS_ERR_OR_NULL(dev)) {
+		pr_err("%s: dev pointer is invalid\n", __func__);
+		return -EINVAL;
+	}
+
+	if (IS_ERR_OR_NULL(sg)) {
+		pr_err("%s: sg table pointer is invalid\n", __func__);
+		return -EINVAL;
+	}
+
+	if (IS_ERR_OR_NULL(dma_buf)) {
+		pr_err("%s: dma_buf pointer is invalid\n", __func__);
+		return -EINVAL;
+	}
+
+	ret = __msm_dma_map_sg(dev, sg, nents, dir, dma_buf, attrs);
+
+	return ret;
+}
+EXPORT_SYMBOL(msm_dma_map_sg_attrs);
+
+static void msm_iommu_meta_destroy(struct kref *kref)
+{
+	struct msm_iommu_meta *meta = container_of(kref, struct msm_iommu_meta,
+						ref);
+
+	if (!list_empty(&meta->iommu_maps)) {
+		WARN(1, "%s: DMA Buffer %pK being destroyed with outstanding iommu mappings!\n",
+		     __func__, meta->buffer);
+	}
+	rb_erase(&meta->node, &iommu_root);
+	kfree(meta);
+}
+
+static void msm_iommu_meta_put(struct msm_iommu_meta *meta)
+{
+	/*
+	 * Need to lock here to prevent race against map/unmap
+	 */
+	mutex_lock(&msm_iommu_map_mutex);
+	kref_put(&meta->ref, msm_iommu_meta_destroy);
+	mutex_unlock(&msm_iommu_map_mutex);
+}
+
+static void msm_iommu_map_release(struct kref *kref)
+{
+	struct msm_iommu_map *map = container_of(kref, struct msm_iommu_map,
+						ref);
+	struct sg_table table;
+
+	table.nents = table.orig_nents = map->nents;
+	table.sgl = map->sgl;
+	list_del(&map->lnode);
+
+	/* Skip an additional cache maintenance on the dma unmap path */
+	if (!(map->attrs & DMA_ATTR_SKIP_CPU_SYNC))
+		map->attrs |= DMA_ATTR_SKIP_CPU_SYNC;
+	dma_unmap_sg_attrs(map->dev, map->sgl, map->nents, map->dir,
+			map->attrs);
+	sg_free_table(&table);
+	kfree(map);
+}
+
+void msm_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl,
+			    int nents, enum dma_data_direction dir,
+			    struct dma_buf *dma_buf, unsigned long attrs)
+{
+	struct msm_iommu_map *iommu_map;
+	struct msm_iommu_meta *meta;
+
+	mutex_lock(&msm_iommu_map_mutex);
+	meta = msm_iommu_meta_lookup(dma_buf->priv);
+	if (!meta) {
+		WARN(1, "%s: (%pK) was never mapped\n", __func__, dma_buf);
+		mutex_unlock(&msm_iommu_map_mutex);
+		goto out;
+
+	}
+	mutex_unlock(&msm_iommu_map_mutex);
+
+	mutex_lock(&meta->lock);
+	iommu_map = msm_iommu_lookup(meta, dev);
+
+	if (!iommu_map) {
+		WARN(1, "%s: (%pK) was never mapped for device  %p\n", __func__,
+				dma_buf, dev);
+		mutex_unlock(&meta->lock);
+		goto out;
+	}
+
+	if (dir != iommu_map->dir)
+		WARN(1, "%s: (%pK) dir:%d differs from original dir:%d\n",
+		     __func__, dma_buf, dir, iommu_map->dir);
+
+	if (attrs && ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0))
+		dma_sync_sg_for_cpu(dev, iommu_map->sgl, iommu_map->nents, dir);
+
+	iommu_map->attrs = attrs;
+	kref_put(&iommu_map->ref, msm_iommu_map_release);
+	mutex_unlock(&meta->lock);
+
+	msm_iommu_meta_put(meta);
+
+out:
+	return;
+}
+EXPORT_SYMBOL(msm_dma_unmap_sg_attrs);
+
+int msm_dma_unmap_all_for_dev(struct device *dev)
+{
+	int ret = 0;
+	struct msm_iommu_meta *meta;
+	struct rb_root *root;
+	struct rb_node *meta_node;
+
+	mutex_lock(&msm_iommu_map_mutex);
+	root = &iommu_root;
+	meta_node = rb_first(root);
+	while (meta_node) {
+		struct msm_iommu_map *iommu_map;
+		struct msm_iommu_map *iommu_map_next;
+
+		meta = rb_entry(meta_node, struct msm_iommu_meta, node);
+		mutex_lock(&meta->lock);
+		list_for_each_entry_safe(iommu_map, iommu_map_next,
+						&meta->iommu_maps, lnode)
+			if (iommu_map->dev == dev)
+				if (!kref_put(&iommu_map->ref,
+						msm_iommu_map_release))
+					ret = -EINVAL;
+
+		mutex_unlock(&meta->lock);
+		meta_node = rb_next(meta_node);
+	}
+	mutex_unlock(&msm_iommu_map_mutex);
+
+	return ret;
+}
+EXPORT_SYMBOL(msm_dma_unmap_all_for_dev);
+
+/*
+ * Only to be called by ION code when a buffer is freed
+ */
+void msm_dma_buf_freed(void *buffer)
+{
+	struct msm_iommu_map *iommu_map;
+	struct msm_iommu_map *iommu_map_next;
+	struct msm_iommu_meta *meta;
+
+	mutex_lock(&msm_iommu_map_mutex);
+	meta = msm_iommu_meta_lookup(buffer);
+	if (!meta) {
+		/* Already unmapped (assuming no late unmapping) */
+		mutex_unlock(&msm_iommu_map_mutex);
+		return;
+	}
+	mutex_unlock(&msm_iommu_map_mutex);
+
+	mutex_lock(&meta->lock);
+
+	list_for_each_entry_safe(iommu_map, iommu_map_next, &meta->iommu_maps,
+				 lnode)
+		kref_put(&iommu_map->ref, msm_iommu_map_release);
+
+	if (!list_empty(&meta->iommu_maps)) {
+		WARN(1, "%s: DMA buffer %pK destroyed with outstanding iommu mappings\n",
+		     __func__, meta->buffer);
+	}
+
+	INIT_LIST_HEAD(&meta->iommu_maps);
+	mutex_unlock(&meta->lock);
+
+	msm_iommu_meta_put(meta);
+}
+EXPORT_SYMBOL(msm_dma_buf_freed);
+
+MODULE_LICENSE("GPL v2");

include/linux/msm_dma_iommu_mapping.h

@@ -0,0 +1,144 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2015-2016, 2018, 2021 The Linux Foundation. All rights reserved.
+ */
+
+#ifndef _LINUX_MSM_DMA_IOMMU_MAPPING_H
+#define _LINUX_MSM_DMA_IOMMU_MAPPING_H
+
+#include <linux/device.h>
+#include <linux/dma-buf.h>
+#include <linux/scatterlist.h>
+#include <linux/dma-mapping.h>
+#include <linux/qcom-dma-mapping.h>
+
+#if IS_ENABLED(CONFIG_QCOM_LAZY_MAPPING)
+/*
+ * This function is not taking a reference to the dma_buf here. It is expected
+ * that clients hold reference to the dma_buf until they are done with mapping
+ * and unmapping.
+ */
+int msm_dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents,
+		   enum dma_data_direction dir, struct dma_buf *dma_buf,
+		   unsigned long attrs);
+
+/*
+ * This function takes an extra reference to the dma_buf.
+ * What this means is that calling msm_dma_unmap_sg will not result in buffer's
+ * iommu mapping being removed, which means that subsequent calls to lazy map
+ * will simply re-use the existing iommu mapping.
+ * The iommu unmapping of the buffer will occur when the ION buffer is
+ * destroyed.
+ * Using lazy mapping can provide a performance benefit because subsequent
+ * mappings are faster.
+ *
+ * The limitation of using this API are that all subsequent iommu mappings
+ * must be the same as the original mapping, ie they must map the same part of
+ * the buffer with the same dma data direction. Also there can't be multiple
+ * mappings of different parts of the buffer.
+ */
+static inline int msm_dma_map_sg_lazy(struct device *dev,
+			       struct scatterlist *sg, int nents,
+			       enum dma_data_direction dir,
+			       struct dma_buf *dma_buf)
+{
+	return msm_dma_map_sg_attrs(dev, sg, nents, dir, dma_buf, 0);
+}
+
+static inline int msm_dma_map_sg(struct device *dev, struct scatterlist *sg,
+				  int nents, enum dma_data_direction dir,
+				  struct dma_buf *dma_buf)
+{
+	unsigned long attrs;
+
+	attrs = DMA_ATTR_NO_DELAYED_UNMAP;
+	return msm_dma_map_sg_attrs(dev, sg, nents, dir, dma_buf, attrs);
+}
+
+void msm_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl,
+			    int nents, enum dma_data_direction dir,
+			    struct dma_buf *dma_buf, unsigned long attrs);
+
+static inline int msm_dma_map_sgtable(struct device *dev, struct sg_table *sgt,
+				      enum dma_data_direction dir,
+				      struct dma_buf *dma_buf, unsigned long attrs)
+{
+	int nents;
+
+	nents = msm_dma_map_sg_attrs(dev, sgt->sgl, sgt->orig_nents, dir, dma_buf, attrs);
+	if (nents <= 0)
+		return -EINVAL;
+
+	sgt->nents = nents;
+	return 0;
+}
+
+static inline void msm_dma_unmap_sgtable(struct device *dev, struct sg_table *sgt,
+					 enum dma_data_direction dir,
+					 struct dma_buf *dma_buf, unsigned long attrs)
+{
+	msm_dma_unmap_sg_attrs(dev, sgt->sgl, sgt->nents, dir, dma_buf, attrs);
+}
+
+int msm_dma_unmap_all_for_dev(struct device *dev);
+
+/*
+ * Below is private function only to be called by framework (ION) and not by
+ * clients.
+ */
+void msm_dma_buf_freed(void *buffer);
+
+#else /*CONFIG_QCOM_LAZY_MAPPING*/
+
+static inline int msm_dma_map_sg_attrs(struct device *dev,
+			struct scatterlist *sg, int nents,
+			enum dma_data_direction dir, struct dma_buf *dma_buf,
+			unsigned long attrs)
+{
+	return -EINVAL;
+}
+
+static inline void
+msm_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl,
+		       int nents, enum dma_data_direction dir,
+		       struct dma_buf *dma_buf, unsigned long attrs)
+{
+}
+
+static inline int msm_dma_map_sg_lazy(struct device *dev,
+			       struct scatterlist *sg, int nents,
+			       enum dma_data_direction dir,
+			       struct dma_buf *dma_buf)
+{
+	return -EINVAL;
+}
+
+static inline int msm_dma_map_sg(struct device *dev, struct scatterlist *sg,
+				  int nents, enum dma_data_direction dir,
+				  struct dma_buf *dma_buf)
+{
+	return -EINVAL;
+}
+
+static inline int msm_dma_map_sgtable(struct device *dev, struct sg_table *sgt,
+				      enum dma_data_direction dir,
+				      struct dma_buf *dma_buf, unsigned long attrs)
+{
+	return -EINVAL;
+}
+
+static inline void msm_dma_unmap_sgtable(struct device *dev, struct sg_table *sgt,
+					 enum dma_data_direction dir,
+					 struct dma_buf *dma_buf, unsigned long attrs)
+{
+}
+
+static inline int msm_dma_unmap_all_for_dev(struct device *dev)
+{
+	return 0;
+}
+
+static inline void msm_dma_buf_freed(void *buffer) {}
+#endif /*CONFIG_QCOM_LAZY_MAPPING*/
+
+#endif

include/linux/qcom_dma_heap.h

@@ -0,0 +1,108 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
+ */
+#ifndef _QCOM_DMA_HEAP_H
+#define _QCOM_DMA_HEAP_H
+
+#include <linux/bits.h>
+#include <linux/bitfield.h>
+#include <linux/dma-buf.h>
+
+#include <linux/qcom_dma_heap_dt_constants.h>
+#include <linux/err.h>
+
+/* Heap flags */
+#define QCOM_DMA_HEAP_FLAG_CACHED	BIT(1)
+
+#define QCOM_DMA_HEAP_FLAG_CP_TRUSTED_VM		BIT(15)
+#define QCOM_DMA_HEAP_FLAG_CP_TZ		BIT(16)
+#define QCOM_DMA_HEAP_FLAG_CP_TOUCH		BIT(17)
+#define QCOM_DMA_HEAP_FLAG_CP_BITSTREAM		BIT(18)
+#define QCOM_DMA_HEAP_FLAG_CP_PIXEL		BIT(19)
+#define QCOM_DMA_HEAP_FLAG_CP_NON_PIXEL		BIT(20)
+#define QCOM_DMA_HEAP_FLAG_CP_CAMERA		BIT(21)
+#define QCOM_DMA_HEAP_FLAG_CP_HLOS		BIT(22)
+#define QCOM_DMA_HEAP_FLAG_CP_SPSS_SP		BIT(23)
+#define QCOM_DMA_HEAP_FLAG_CP_SPSS_SP_SHARED	BIT(24)
+#define QCOM_DMA_HEAP_FLAG_CP_SEC_DISPLAY	BIT(25)
+#define QCOM_DMA_HEAP_FLAG_CP_APP		BIT(26)
+#define QCOM_DMA_HEAP_FLAG_CP_CAMERA_PREVIEW	BIT(27)
+#define QCOM_DMA_HEAP_FLAG_CP_MSS_MSA		BIT(28)
+#define QCOM_DMA_HEAP_FLAG_CP_CDSP		BIT(29)
+#define QCOM_DMA_HEAP_FLAG_CP_SPSS_HLOS_SHARED	BIT(30)
+
+#define QCOM_DMA_HEAP_FLAGS_CP_MASK	GENMASK(30, 15)
+
+#define QCOM_DMA_HEAP_FLAG_SECURE	BIT(31)
+
+/**
+ * carveout_heap_add_memory - Dynamically add reserved heap memory
+ * @heap_name:	The name of the DMA-BUF Heap we're giving memory to
+ * @sgt:	An SGT corresponding to the memory adding to the heap
+ * @cookie:	Cookie for managing refcount.
+ * @get:	Function callback for increasing refcount.
+ * @put:	Function callback for decreasing refcount.
+ *
+ * Return: 0 on success, a negative error value otherwise.
+ */
+int carveout_heap_add_memory(char *heap_name, struct sg_table *sgt, void *cookie,
+			int (*get)(void *), void (*put)(void *));
+
+
+/**
+ * carveout_heap_remove_memory - Dynamically remove reserved heap memory
+ * @heap_name:	The name of the DMA-BUF Heap we're taking memory away from
+ * @sgt:	An SGT corresponding to the memory we're removing from the heap
+ *
+ * Return: 0 on success, a negative error value otherwise.
+ */
+int carveout_heap_remove_memory(char *heap_name, struct sg_table *sgt);
+
+struct dma_buf_heap_prefetch_region {
+	u64 size;
+	struct dma_heap *heap;
+};
+
+int qcom_secure_system_heap_prefetch(struct dma_buf_heap_prefetch_region *regions,
+				     size_t nr_regions);
+
+int qcom_secure_system_heap_drain(struct dma_buf_heap_prefetch_region *regions,
+				  size_t nr_regions);
+
+/**
+ * dma_buf_heap_hyp_assign - wrapper function for hyp-assigning a dma_buf
+ * @buf:		dma_buf to hyp-assign away from HLOS
+ * @dest_vmids:		array of QCOM_DMA_HEAP_FLAG VMIDs (as defined above)
+ * @dest_perms:		array of PERM_READ/PERM_WRITE/PERM_EXEC permission bits (as
+ *			defined in include/soc/qcom/secure_buffer.h), such that
+ *			dest_perms[i] specifies the permissions for VMID dest_vmids[i]
+ * @dest_nelems:	number of elements in dest_vmids and dest_perms
+ *
+ * Return: Temporarily return -EINVAL whilst the functon isn't present. Otherwise, return
+ * 0 on success, or a negative value returned by hyp_assign_table() on failure.
+ */
+static inline int dma_buf_heap_hyp_assign(struct dma_buf *buf, int *dest_vmids, int *dest_perms,
+				   int dest_nelems)
+{
+	return -EINVAL;
+}
+
+/**
+ * dma_buf_heap_hyp_unassign - wrapper function that hyp-assigns a dma_buf back to HLOS
+ * @buf:	dma_buf to hyp-assign back to HLOS
+ *
+ * This function takes a dma_buf, and re-assigns it to HLOS with RWX permissions (at the
+ * S2 level). The end-points to which the buffer was assigned to are tracked by the flags
+ * kept in the msm_heap_helper_buffer->flags field (the helper_buffer is accesed through
+ * dma_buf->priv), so the corresponding VMIDs don't need to be supplied as arguments.
+ *
+ * Return: Temporarily return -EINVAL whilst the functon isn't present. Otherwise, return
+ * 0 on success, or a negative value returned by hyp_assign_table() on failure.
+ */
+static inline int dma_buf_heap_hyp_unassign(struct dma_buf *buf)
+{
+	return -EINVAL;
+}
+
+#endif /* _QCOM_DMA_HEAP_H */

include/linux/qcom_dma_heap_dt_constants.h

@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
+ */
+#ifndef _QCOM_DMA_HEAP_DT_CONSTATS_H
+#define _QCOM_DMA_HEAP_DT_CONSTATS_H
+
+#define HEAP_TYPE_SECURE_CARVEOUT 0
+#define HEAP_TYPE_CARVEOUT 1
+#define HEAP_TYPE_CMA 2
+
+#endif /* _QCOM_DMA_HEAP_DT_CONSTATS_H */

include/uapi/linux/msm_ion.h

@@ -0,0 +1,95 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+/*
+ * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved.
+ */
+#ifndef _UAPI_LINUX_MSM_ION_H
+#define _UAPI_LINUX_MSM_ION_H
+
+#include <linux/types.h>
+#include <linux/msm_ion_ids.h>
+
+/**
+ * TARGET_ION_ABI_VERSION can be used by user space clients to ensure that at
+ * compile time only their code which uses the appropriate ION APIs for
+ * this kernel is included.
+ */
+#define TARGET_ION_ABI_VERSION 3
+
+enum msm_ion_heap_types {
+	ION_HEAP_TYPE_MSM_START = 16,
+	ION_HEAP_TYPE_SECURE_DMA = ION_HEAP_TYPE_MSM_START,
+	ION_HEAP_TYPE_SYSTEM_SECURE,
+	ION_HEAP_TYPE_HYP_CMA,
+	ION_HEAP_TYPE_MSM_CARVEOUT,
+	ION_HEAP_TYPE_SECURE_CARVEOUT,
+	ION_HEAP_TYPE_MSM_SYSTEM,
+};
+
+/**
+ * Flags to be used when allocating from the secure heap for
+ * content protection
+ */
+#define ION_FLAG_CP_TRUSTED_VM		ION_BIT(15)
+/* ION_FLAG_POOL_FORCE_ALLOC uses ION_BIT(16) */
+#define ION_FLAG_CP_TOUCH		ION_BIT(17)
+#define ION_FLAG_CP_BITSTREAM		ION_BIT(18)
+#define ION_FLAG_CP_PIXEL		ION_BIT(19)
+#define ION_FLAG_CP_NON_PIXEL		ION_BIT(20)
+#define ION_FLAG_CP_CAMERA		ION_BIT(21)
+#define ION_FLAG_CP_HLOS		ION_BIT(22)
+#define ION_FLAG_CP_SPSS_SP		ION_BIT(23)
+#define ION_FLAG_CP_SPSS_SP_SHARED	ION_BIT(24)
+#define ION_FLAG_CP_SEC_DISPLAY		ION_BIT(25)
+#define ION_FLAG_CP_APP			ION_BIT(26)
+#define ION_FLAG_CP_CAMERA_PREVIEW	ION_BIT(27)
+/* ION_FLAG_ALLOW_NON_CONTIG uses ION_BIT(28) */
+#define ION_FLAG_CP_CDSP		ION_BIT(29)
+#define ION_FLAG_CP_SPSS_HLOS_SHARED	ION_BIT(30)
+
+#define ION_FLAGS_CP_MASK	0x6FFE8000
+
+/**
+ * Flag to allow non continguous allocation of memory from secure
+ * heap
+ */
+#define ION_FLAG_ALLOW_NON_CONTIG       ION_BIT(28)
+
+/**
+ * Flag to use when allocating to indicate that a heap is secure.
+ * Do NOT use BIT macro since it is defined in #ifdef __KERNEL__
+ */
+#define ION_FLAG_SECURE			ION_BIT(31)
+
+/*
+ * Used in conjunction with heap which pool memory to force an allocation
+ * to come from the page allocator directly instead of from the pool allocation
+ */
+#define ION_FLAG_POOL_FORCE_ALLOC	ION_BIT(16)
+
+/**
+ * Macro should be used with ion_heap_ids defined above.
+ */
+#define ION_HEAP(bit)			bit
+
+#define ION_IOC_MSM_MAGIC 'M'
+
+struct ion_prefetch_regions {
+	__u64 sizes;
+	__u32 vmid;
+	__u32 nr_sizes;
+};
+
+struct ion_prefetch_data {
+	__u64 len;
+	__u64 regions;
+	__u32 heap_id;
+	__u32 nr_regions;
+};
+
+#define ION_IOC_PREFETCH		_IOWR(ION_IOC_MSM_MAGIC, 3, \
+						struct ion_prefetch_data)
+
+#define ION_IOC_DRAIN			_IOWR(ION_IOC_MSM_MAGIC, 4, \
+						struct ion_prefetch_data)
+
+#endif /* _UAPI_LINUX_MSM_ION_H */

include/uapi/linux/msm_ion_ids.h

@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: GPL-2.0-only WITH Linux-syscall-note */
+/*
+ * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved.
+ */
+#ifndef _MSM_ION_IDS_H
+#define _MSM_ION_IDS_H
+
+#define ION_BIT(nr) (1U << (nr))
+
+/**
+ * These are the only ids that should be used for Ion heap ids.
+ * The ids listed are the order in which allocation will be attempted
+ * if specified. Don't swap the order of heap ids unless you know what
+ * you are doing!
+ * Id's are spaced by purpose to allow new Id's to be inserted in-between (for
+ * possible fallbacks)
+ */
+
+/* ION_BIT(0) is reserved for the generic system heap. */
+#define ION_QSECOM_TA_HEAP_ID		ION_BIT(1)
+#define ION_CAMERA_HEAP_ID		ION_BIT(2)
+#define ION_DISPLAY_HEAP_ID		ION_BIT(3)
+#define ION_ADSP_HEAP_ID		ION_BIT(4)
+#define ION_AUDIO_ML_HEAP_ID		ION_BIT(5)
+#define ION_USER_CONTIG_HEAP_ID		ION_BIT(6)
+#define ION_QSECOM_HEAP_ID		ION_BIT(7)
+#define ION_AUDIO_HEAP_ID		ION_BIT(8)
+#define ION_CP_MM_HEAP_ID		ION_BIT(9)
+#define ION_SECURE_HEAP_ID		ION_BIT(10)
+#define ION_SECURE_DISPLAY_HEAP_ID	ION_BIT(11)
+#define ION_SPSS_HEAP_ID		ION_BIT(14)
+#define ION_SECURE_CARVEOUT_HEAP_ID	ION_BIT(15)
+#define ION_TUI_CARVEOUT_HEAP_ID	ION_BIT(16)
+#define ION_SYSTEM_HEAP_ID		ION_BIT(25)
+#define ION_HEAP_ID_RESERVED		ION_BIT(31)
+
+#endif /* _MSM_ION_IDS_H */