mirror of
https://github.com/torvalds/linux.git
synced 2026-07-31 03:27:03 +02:00
dma-heap: qcom: Take a snapshot of our DMA-BUF heap driver from msm-5.15
Take a snapshot of our DMA-BUF heap driver as of commit f843a80d2c45
("Merge "defconfig: gen3auto: Separated if_changed"") on msm-5.15.
On top of taking the snapshot, make the following changes:
- Remove the references to the deferred free helper, which will be
added later.
- Remove the references to the pool size getters, which will be
added later.
- Add the deferred mapping driver.
- Remove the heap helper buffer, and associated headers / inclusions
(these are unused).
- Remove the bitstream contig heap as this is no longer required.
Change-Id: Ib98b08139d11940f4f0030294c6c7c18897f6531
Signed-off-by: Chris Goldsworthy <quic_cgoldswo@quicinc.com>
This commit is contained in:
Chris Goldsworthy
parent
e117a3efa4
commit
3a684e8037
|
|
@ -12,3 +12,66 @@ config DMABUF_HEAPS_CMA
|
|||
Choose this option to enable dma-buf CMA heap. This heap is backed
|
||||
by the Contiguous Memory Allocator (CMA). If your system has these
|
||||
regions, you should say Y here.
|
||||
|
||||
config QCOM_DMABUF_HEAPS
|
||||
tristate "QCOM DMA-BUF Heaps"
|
||||
depends on DMABUF_HEAPS
|
||||
help
|
||||
Choose this option to build the dma-buf heap module. Other
|
||||
heaps can subsequently be compiled into this module by enabling
|
||||
the appropriate defconfig option.
|
||||
If in doubt, say M here.
|
||||
|
||||
config QCOM_DMABUF_HEAPS_SYSTEM
|
||||
bool "QCOM DMA-BUF System Heap"
|
||||
depends on QCOM_DMABUF_HEAPS
|
||||
help
|
||||
Choose this option to build the QCOM DMA-BUF system
|
||||
heap. This heap combines the built-in system heap
|
||||
along with other optimizatons. If in doubt, say Y
|
||||
here.
|
||||
|
||||
config QCOM_DMABUF_HEAPS_PAGE_POOL_REFILL
|
||||
bool "QCOM DMA-BUF Page Pool Refill"
|
||||
depends on QCOM_DMABUF_HEAPS_SYSTEM
|
||||
help
|
||||
Choose this option to enable the page pool auto refill
|
||||
feature for the system heap's page pools. When enabled,
|
||||
a thread is created which wakes up to refill the page pools
|
||||
when they fall below a particular threshold, which in turn
|
||||
results in smaller allocation latencies. If in doubt, say Y
|
||||
here.
|
||||
|
||||
config QCOM_DMABUF_HEAPS_SYSTEM_SECURE
|
||||
bool "QCOM DMA-BUF System Secure Heap"
|
||||
depends on QCOM_DMABUF_HEAPS && QCOM_SECURE_BUFFER
|
||||
depends on QCOM_DMABUF_HEAPS_SYSTEM
|
||||
help
|
||||
Choose this option to build the QCOM DMA-BUF system
|
||||
secure heap. This heap, which combines the built-in
|
||||
system heap along with other optimizations, assigns
|
||||
memory that is secure by default. If in doubt, say Y
|
||||
here.
|
||||
|
||||
config QCOM_DMABUF_HEAPS_SYSTEM_UNCACHED
|
||||
bool "QCOM DMA-BUF Uncached System Heap"
|
||||
depends on QCOM_DMABUF_HEAPS && QCOM_DMABUF_HEAPS_SYSTEM
|
||||
help
|
||||
Choose this option to create an uncached QCOM DMA-BUF
|
||||
system heap. This heap is equivalent to our system heap,
|
||||
such that the memory is mapped as uncached. If in doubt,
|
||||
say Y here.
|
||||
|
||||
config QCOM_DMABUF_HEAPS_CMA
|
||||
bool "QCOM DMA-BUF CMA Heap"
|
||||
depends on QCOM_DMABUF_HEAPS && DMA_CMA
|
||||
help
|
||||
Choose this option to build the QCOM DMA-BUF CMA heap.
|
||||
If in doubt, say Y here.
|
||||
|
||||
config QCOM_DMABUF_HEAPS_CARVEOUT
|
||||
bool "QCOM dma-buf Carveout Heap"
|
||||
depends on QCOM_DMABUF_HEAPS
|
||||
help
|
||||
Choose this option to build the QCOM DMA-BUF Carveout heap.
|
||||
If in doubt, say Y here.
|
||||
|
|
|
|||
|
|
@ -1,3 +1,14 @@
|
|||
# SPDX-License-Identifier: GPL-2.0
|
||||
obj-$(CONFIG_DMABUF_HEAPS_SYSTEM) += system_heap.o
|
||||
obj-$(CONFIG_DMABUF_HEAPS_CMA) += cma_heap.o
|
||||
obj-$(CONFIG_QCOM_DMABUF_HEAPS) += qcom_dma_heaps.o
|
||||
qcom_dma_heaps-y := qcom_dma_heap.o qcom_dt_parser.o qcom_sg_ops.o
|
||||
qcom_dma_heaps-$(CONFIG_QCOM_DMABUF_HEAPS_SYSTEM) += qcom_system_heap.o \
|
||||
qcom_dynamic_page_pool.o
|
||||
qcom_dma_heaps-$(CONFIG_QCOM_DMABUF_HEAPS_SYSTEM_SECURE) += qcom_secure_system_heap.o \
|
||||
qcom_dynamic_page_pool.o \
|
||||
qcom_dma_heap_secure_utils.o
|
||||
qcom_dma_heaps-$(CONFIG_QCOM_DMABUF_HEAPS_CMA) += qcom_cma_heap.o
|
||||
qcom_dma_heaps-$(CONFIG_QCOM_DMABUF_HEAPS_CARVEOUT) += qcom_carveout_heap.o \
|
||||
qcom_dma_heap_secure_utils.o
|
||||
qcom_dma_heaps-$(CONFIG_QCOM_DMABUF_HEAPS_BITSTREAM_CONTIG) += qcom_bitstream_contig_heap.o
|
||||
|
|
|
|||
589
drivers/dma-buf/heaps/qcom_carveout_heap.c
Normal file
589
drivers/dma-buf/heaps/qcom_carveout_heap.c
Normal file
|
|
@ -0,0 +1,589 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/*
|
||||
* DMA-BUF heap carveout heap allocator. Copied from
|
||||
* drivers/staging/android/ion/heaps/ion_carveout_heap.c as of commit
|
||||
* aeb022cc01ecc ("dma-heap: qcom: Change symbol names to let module be built
|
||||
* in")
|
||||
*
|
||||
* Copyright (C) 2011 Google, Inc.
|
||||
* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <linux/dma-mapping.h>
|
||||
#include <linux/err.h>
|
||||
#include <linux/genalloc.h>
|
||||
#include <linux/io.h>
|
||||
#include <linux/mm.h>
|
||||
#include <linux/scatterlist.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/vmalloc.h>
|
||||
#include <soc/qcom/secure_buffer.h>
|
||||
#include <linux/of.h>
|
||||
#include <linux/of_address.h>
|
||||
#include <linux/list.h>
|
||||
#include <linux/dma-buf.h>
|
||||
#include <linux/dma-heap.h>
|
||||
#include <linux/qcom_dma_heap.h>
|
||||
|
||||
#include "qcom_dma_heap_secure_utils.h"
|
||||
#include "qcom_sg_ops.h"
|
||||
#include "qcom_carveout_heap.h"
|
||||
|
||||
#define CARVEOUT_ALLOCATE_FAIL -1
|
||||
|
||||
|
||||
/*
|
||||
* @pool_refcount_priv -
|
||||
* Cookie set by carveout_heap_add_memory for use with its callbacks.
|
||||
* Cookie provider will call carveout_heap_remove_memory if refcount
|
||||
* reaches zero.
|
||||
* @pool_refcount_get -
|
||||
* Function callback to Increase refcount. Returns 0
|
||||
* on success and fails if refcount is already zero.
|
||||
* @pool_refcount_put - Function callback to decrease refcount.
|
||||
*/
|
||||
struct carveout_heap {
|
||||
struct dma_heap *heap;
|
||||
struct rw_semaphore mem_sem;
|
||||
struct gen_pool *pool;
|
||||
struct device *dev;
|
||||
bool is_secure;
|
||||
bool is_nomap;
|
||||
phys_addr_t base;
|
||||
|
||||
void *pool_refcount_priv;
|
||||
int (*pool_refcount_get)(void *priv);
|
||||
void (*pool_refcount_put)(void *priv);
|
||||
};
|
||||
|
||||
struct secure_carveout_heap {
|
||||
u32 token;
|
||||
struct carveout_heap carveout_heap;
|
||||
};
|
||||
|
||||
static void sc_heap_free(struct qcom_sg_buffer *buffer);
|
||||
|
||||
void pages_sync_for_device(struct device *dev, struct page *page,
|
||||
size_t size, enum dma_data_direction dir)
|
||||
{
|
||||
struct scatterlist sg;
|
||||
|
||||
sg_init_table(&sg, 1);
|
||||
sg_set_page(&sg, page, size, 0);
|
||||
/*
|
||||
* This is not correct - sg_dma_address needs a dma_addr_t that is valid
|
||||
* for the targeted device, but this works on the currently targeted
|
||||
* hardware.
|
||||
*/
|
||||
sg_dma_address(&sg) = page_to_phys(page);
|
||||
dma_sync_sg_for_device(dev, &sg, 1, dir);
|
||||
}
|
||||
|
||||
static int carveout_pool_refcount_get(struct carveout_heap *carveout_heap)
|
||||
{
|
||||
if (!carveout_heap->pool_refcount_get)
|
||||
return 0;
|
||||
|
||||
return carveout_heap->pool_refcount_get(carveout_heap->pool_refcount_priv);
|
||||
}
|
||||
|
||||
static void carveout_pool_refcount_put(struct carveout_heap *carveout_heap)
|
||||
{
|
||||
if (!carveout_heap->pool_refcount_put)
|
||||
return;
|
||||
|
||||
carveout_heap->pool_refcount_put(carveout_heap->pool_refcount_priv);
|
||||
}
|
||||
|
||||
static phys_addr_t carveout_allocate(struct carveout_heap *carveout_heap,
|
||||
unsigned long size)
|
||||
{
|
||||
unsigned long offset = CARVEOUT_ALLOCATE_FAIL;
|
||||
|
||||
down_read(&carveout_heap->mem_sem);
|
||||
if (carveout_heap->pool) {
|
||||
if (carveout_pool_refcount_get(carveout_heap))
|
||||
goto unlock;
|
||||
|
||||
offset = gen_pool_alloc(carveout_heap->pool, size);
|
||||
if (!offset) {
|
||||
offset = CARVEOUT_ALLOCATE_FAIL;
|
||||
carveout_pool_refcount_put(carveout_heap);
|
||||
goto unlock;
|
||||
}
|
||||
}
|
||||
|
||||
unlock:
|
||||
up_read(&carveout_heap->mem_sem);
|
||||
return offset;
|
||||
}
|
||||
|
||||
static void carveout_free(struct carveout_heap *carveout_heap,
|
||||
phys_addr_t addr, unsigned long size)
|
||||
{
|
||||
if (addr == CARVEOUT_ALLOCATE_FAIL)
|
||||
return;
|
||||
|
||||
down_read(&carveout_heap->mem_sem);
|
||||
if (carveout_heap->pool)
|
||||
gen_pool_free(carveout_heap->pool, addr, size);
|
||||
carveout_pool_refcount_put(carveout_heap);
|
||||
up_read(&carveout_heap->mem_sem);
|
||||
}
|
||||
|
||||
struct mem_buf_vmperm *
|
||||
carveout_setup_vmperm(struct carveout_heap *carveout_heap,
|
||||
struct sg_table *sgt)
|
||||
{
|
||||
struct secure_carveout_heap *sc_heap;
|
||||
struct mem_buf_vmperm *vmperm;
|
||||
int *vmids, *perms;
|
||||
u32 nr;
|
||||
int ret;
|
||||
|
||||
if (!carveout_heap->is_secure) {
|
||||
vmperm = mem_buf_vmperm_alloc(sgt);
|
||||
return vmperm;
|
||||
}
|
||||
|
||||
sc_heap = container_of(carveout_heap,
|
||||
struct secure_carveout_heap, carveout_heap);
|
||||
|
||||
ret = get_vmperm_from_ion_flags(sc_heap->token,
|
||||
&vmids, &perms, &nr);
|
||||
if (ret)
|
||||
return ERR_PTR(ret);
|
||||
|
||||
vmperm = mem_buf_vmperm_alloc_staticvm(sgt, vmids, perms, nr);
|
||||
kfree(vmids);
|
||||
kfree(perms);
|
||||
|
||||
return vmperm;
|
||||
}
|
||||
|
||||
static struct dma_buf *__carveout_heap_allocate(struct carveout_heap *carveout_heap,
|
||||
unsigned long len,
|
||||
unsigned long fd_flags,
|
||||
unsigned long heap_flags,
|
||||
void (*buffer_free)(struct qcom_sg_buffer *))
|
||||
{
|
||||
struct sg_table *table;
|
||||
struct qcom_sg_buffer *buffer;
|
||||
phys_addr_t paddr;
|
||||
int ret;
|
||||
DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
|
||||
struct dma_buf *dmabuf;
|
||||
struct device *dev = carveout_heap->dev;
|
||||
|
||||
buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
|
||||
if (!buffer)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
/* Initialize the buffer */
|
||||
INIT_LIST_HEAD(&buffer->attachments);
|
||||
mutex_init(&buffer->lock);
|
||||
buffer->heap = carveout_heap->heap;
|
||||
buffer->len = len;
|
||||
buffer->free = buffer_free;
|
||||
if (carveout_heap->is_nomap)
|
||||
buffer->uncached = true;
|
||||
|
||||
table = &buffer->sg_table;
|
||||
ret = sg_alloc_table(table, 1, GFP_KERNEL);
|
||||
if (ret)
|
||||
goto err_free;
|
||||
|
||||
paddr = carveout_allocate(carveout_heap, len);
|
||||
if (paddr == CARVEOUT_ALLOCATE_FAIL) {
|
||||
ret = -ENOMEM;
|
||||
goto err_free_table;
|
||||
}
|
||||
|
||||
sg_set_page(table->sgl, pfn_to_page(PFN_DOWN(paddr)), len, 0);
|
||||
|
||||
if (!carveout_heap->is_secure && !carveout_heap->is_nomap)
|
||||
pages_sync_for_device(dev, sg_page(table->sgl),
|
||||
buffer->len, DMA_FROM_DEVICE);
|
||||
|
||||
buffer->vmperm = carveout_setup_vmperm(carveout_heap, &buffer->sg_table);
|
||||
if (IS_ERR(buffer->vmperm))
|
||||
goto err_free_carveout;
|
||||
|
||||
|
||||
/* Instantiate our dma_buf */
|
||||
exp_info.exp_name = dma_heap_get_name(carveout_heap->heap);
|
||||
exp_info.size = buffer->len;
|
||||
exp_info.flags = fd_flags;
|
||||
exp_info.priv = buffer;
|
||||
dmabuf = mem_buf_dma_buf_export(&exp_info, &qcom_sg_buf_ops);
|
||||
if (IS_ERR(dmabuf)) {
|
||||
ret = PTR_ERR(dmabuf);
|
||||
goto err_free_vmperm;
|
||||
}
|
||||
|
||||
return dmabuf;
|
||||
|
||||
err_free_vmperm:
|
||||
mem_buf_vmperm_release(buffer->vmperm);
|
||||
err_free_carveout:
|
||||
carveout_free(carveout_heap, paddr, len);
|
||||
err_free_table:
|
||||
sg_free_table(table);
|
||||
err_free:
|
||||
kfree(buffer);
|
||||
return ERR_PTR(ret);
|
||||
}
|
||||
|
||||
static int carveout_pages_zero(struct page *page, size_t size, pgprot_t prot);
|
||||
|
||||
static void carveout_heap_free(struct qcom_sg_buffer *buffer)
|
||||
{
|
||||
struct carveout_heap *carveout_heap;
|
||||
struct sg_table *table = &buffer->sg_table;
|
||||
struct page *page = sg_page(table->sgl);
|
||||
phys_addr_t paddr = page_to_phys(page);
|
||||
struct device *dev;
|
||||
|
||||
carveout_heap = dma_heap_get_drvdata(buffer->heap);
|
||||
|
||||
dev = carveout_heap->dev;
|
||||
|
||||
if (!carveout_heap->is_nomap) {
|
||||
carveout_pages_zero(page, buffer->len, PAGE_KERNEL);
|
||||
pages_sync_for_device(dev, page,
|
||||
buffer->len, DMA_BIDIRECTIONAL);
|
||||
} else {
|
||||
carveout_pages_zero(page, buffer->len, pgprot_writecombine(PAGE_KERNEL));
|
||||
}
|
||||
|
||||
carveout_free(carveout_heap, paddr, buffer->len);
|
||||
sg_free_table(table);
|
||||
kfree(buffer);
|
||||
}
|
||||
|
||||
|
||||
static struct dma_buf *carveout_heap_allocate(struct dma_heap *heap,
|
||||
unsigned long len,
|
||||
unsigned long fd_flags,
|
||||
unsigned long heap_flags)
|
||||
{
|
||||
struct carveout_heap *carveout_heap = dma_heap_get_drvdata(heap);
|
||||
|
||||
return __carveout_heap_allocate(carveout_heap, len, fd_flags,
|
||||
heap_flags, carveout_heap_free);
|
||||
}
|
||||
|
||||
static int carveout_heap_clear_pages(struct page **pages, int num, pgprot_t prot)
|
||||
{
|
||||
void *addr = vmap(pages, num, VM_MAP, prot);
|
||||
|
||||
if (!addr)
|
||||
return -ENOMEM;
|
||||
memset(addr, 0, PAGE_SIZE * num);
|
||||
vunmap(addr);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int carveout_heap_sglist_zero(struct scatterlist *sgl, unsigned int nents, pgprot_t prot)
|
||||
{
|
||||
int p = 0;
|
||||
int ret = 0;
|
||||
struct sg_page_iter piter;
|
||||
struct page *pages[32];
|
||||
|
||||
for_each_sg_page(sgl, &piter, nents, 0) {
|
||||
pages[p++] = sg_page_iter_page(&piter);
|
||||
if (p == ARRAY_SIZE(pages)) {
|
||||
ret = carveout_heap_clear_pages(pages, p, prot);
|
||||
if (ret)
|
||||
return ret;
|
||||
p = 0;
|
||||
}
|
||||
}
|
||||
if (p)
|
||||
ret = carveout_heap_clear_pages(pages, p, prot);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int carveout_pages_zero(struct page *page, size_t size, pgprot_t prot)
|
||||
{
|
||||
struct scatterlist sg;
|
||||
|
||||
sg_init_table(&sg, 1);
|
||||
sg_set_page(&sg, page, size, 0);
|
||||
return carveout_heap_sglist_zero(&sg, 1, prot);
|
||||
}
|
||||
|
||||
static int carveout_init_heap_memory(struct carveout_heap *co_heap,
|
||||
phys_addr_t base, ssize_t size,
|
||||
bool sync)
|
||||
{
|
||||
struct page *page = pfn_to_page(PFN_DOWN(base));
|
||||
struct device *dev = co_heap->dev;
|
||||
int ret = 0;
|
||||
|
||||
if (sync) {
|
||||
if (!pfn_valid(PFN_DOWN(base)))
|
||||
return -EINVAL;
|
||||
pages_sync_for_device(dev, page, size, DMA_BIDIRECTIONAL);
|
||||
}
|
||||
|
||||
ret = carveout_pages_zero(page, size, pgprot_writecombine(PAGE_KERNEL));
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
co_heap->pool = gen_pool_create(PAGE_SHIFT, -1);
|
||||
if (!co_heap->pool)
|
||||
return -ENOMEM;
|
||||
|
||||
co_heap->base = base;
|
||||
gen_pool_add(co_heap->pool, co_heap->base, size, -1);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int carveout_heap_add_memory(char *heap_name, struct sg_table *sgt, void *cookie,
|
||||
int (*get)(void *), void (*put)(void *))
|
||||
{
|
||||
struct dma_heap *heap;
|
||||
struct carveout_heap *carveout_heap;
|
||||
int ret;
|
||||
|
||||
if (!sgt || sgt->nents != 1)
|
||||
return -EINVAL;
|
||||
|
||||
heap = dma_heap_find(heap_name);
|
||||
if (!heap) {
|
||||
pr_err_ratelimited("%s: No heap named %s\n", __func__, heap_name);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (!get || !put || !cookie) {
|
||||
pr_err_ratelimited("%s: Missing refcount callbacks\n", __func__);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
carveout_heap = dma_heap_get_drvdata(heap);
|
||||
|
||||
down_write(&carveout_heap->mem_sem);
|
||||
if (carveout_heap->pool) {
|
||||
ret = -EBUSY;
|
||||
goto unlock;
|
||||
}
|
||||
|
||||
ret = carveout_init_heap_memory(carveout_heap,
|
||||
page_to_phys(sg_page(sgt->sgl)),
|
||||
sgt->sgl->length, true);
|
||||
if (ret)
|
||||
goto unlock;
|
||||
|
||||
carveout_heap->pool_refcount_priv = cookie;
|
||||
carveout_heap->pool_refcount_get = get;
|
||||
carveout_heap->pool_refcount_put = put;
|
||||
|
||||
unlock:
|
||||
up_write(&carveout_heap->mem_sem);
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL(carveout_heap_add_memory);
|
||||
|
||||
int carveout_heap_remove_memory(char *heap_name,
|
||||
struct sg_table *sgt)
|
||||
{
|
||||
struct dma_heap *heap;
|
||||
struct carveout_heap *carveout_heap;
|
||||
phys_addr_t base;
|
||||
int ret = 0;
|
||||
|
||||
if (!sgt || sgt->nents != 1)
|
||||
return -EINVAL;
|
||||
|
||||
heap = dma_heap_find(heap_name);
|
||||
if (!heap)
|
||||
return -EINVAL;
|
||||
|
||||
carveout_heap = dma_heap_get_drvdata(heap);
|
||||
|
||||
down_write(&carveout_heap->mem_sem);
|
||||
if (!carveout_heap->pool) {
|
||||
ret = -EINVAL;
|
||||
goto unlock;
|
||||
}
|
||||
|
||||
base = page_to_phys(sg_page(sgt->sgl));
|
||||
if (carveout_heap->base != base) {
|
||||
ret = 0;
|
||||
goto unlock;
|
||||
}
|
||||
|
||||
if (gen_pool_size(carveout_heap->pool) !=
|
||||
gen_pool_avail(carveout_heap->pool)) {
|
||||
ret = -EBUSY;
|
||||
goto unlock;
|
||||
}
|
||||
|
||||
gen_pool_destroy(carveout_heap->pool);
|
||||
carveout_heap->pool = NULL;
|
||||
carveout_heap->base = 0;
|
||||
carveout_heap->pool_refcount_priv = NULL;
|
||||
carveout_heap->pool_refcount_get = NULL;
|
||||
carveout_heap->pool_refcount_put = NULL;
|
||||
unlock:
|
||||
up_write(&carveout_heap->mem_sem);
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL(carveout_heap_remove_memory);
|
||||
|
||||
static int __carveout_heap_init(struct platform_heap *heap_data,
|
||||
struct carveout_heap *carveout_heap,
|
||||
bool sync)
|
||||
{
|
||||
struct device *dev = heap_data->dev;
|
||||
bool dynamic_heap = heap_data->is_dynamic;
|
||||
int ret = 0;
|
||||
|
||||
carveout_heap->dev = dev;
|
||||
if (!dynamic_heap)
|
||||
ret = carveout_init_heap_memory(carveout_heap,
|
||||
heap_data->base,
|
||||
heap_data->size, sync);
|
||||
|
||||
init_rwsem(&carveout_heap->mem_sem);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static const struct dma_heap_ops carveout_heap_ops = {
|
||||
.allocate = carveout_heap_allocate,
|
||||
};
|
||||
|
||||
static void carveout_heap_destroy(struct carveout_heap *heap);
|
||||
|
||||
int qcom_carveout_heap_create(struct platform_heap *heap_data)
|
||||
{
|
||||
struct dma_heap_export_info exp_info;
|
||||
struct carveout_heap *carveout_heap;
|
||||
int ret;
|
||||
|
||||
carveout_heap = kzalloc(sizeof(*carveout_heap), GFP_KERNEL);
|
||||
if (!carveout_heap)
|
||||
return -ENOMEM;
|
||||
|
||||
ret = __carveout_heap_init(heap_data, carveout_heap, !heap_data->is_nomap);
|
||||
if (ret)
|
||||
goto err;
|
||||
|
||||
carveout_heap->is_secure = false;
|
||||
carveout_heap->is_nomap = heap_data->is_nomap;
|
||||
|
||||
exp_info.name = heap_data->name;
|
||||
exp_info.ops = &carveout_heap_ops;
|
||||
exp_info.priv = carveout_heap;
|
||||
|
||||
carveout_heap->heap = dma_heap_add(&exp_info);
|
||||
if (IS_ERR(carveout_heap->heap)) {
|
||||
ret = PTR_ERR(carveout_heap->heap);
|
||||
goto destroy_heap;
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
||||
destroy_heap:
|
||||
carveout_heap_destroy(carveout_heap);
|
||||
err:
|
||||
kfree(carveout_heap);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void carveout_heap_destroy(struct carveout_heap *carveout_heap)
|
||||
{
|
||||
down_write(&carveout_heap->mem_sem);
|
||||
if (carveout_heap->pool)
|
||||
gen_pool_destroy(carveout_heap->pool);
|
||||
up_write(&carveout_heap->mem_sem);
|
||||
carveout_heap = NULL;
|
||||
}
|
||||
|
||||
static struct dma_buf *sc_heap_allocate(struct dma_heap *heap,
|
||||
unsigned long len,
|
||||
unsigned long fd_flags,
|
||||
unsigned long heap_flags)
|
||||
{
|
||||
struct secure_carveout_heap *sc_heap;
|
||||
|
||||
sc_heap = dma_heap_get_drvdata(heap);
|
||||
return __carveout_heap_allocate(&sc_heap->carveout_heap, len,
|
||||
fd_flags, heap_flags, sc_heap_free);
|
||||
}
|
||||
|
||||
static void sc_heap_free(struct qcom_sg_buffer *buffer)
|
||||
{
|
||||
struct secure_carveout_heap *sc_heap;
|
||||
struct sg_table *table = &buffer->sg_table;
|
||||
struct page *page = sg_page(table->sgl);
|
||||
phys_addr_t paddr = PFN_PHYS(page_to_pfn(page));
|
||||
|
||||
sc_heap = dma_heap_get_drvdata(buffer->heap);
|
||||
|
||||
if (qcom_is_buffer_hlos_accessible(sc_heap->token)) {
|
||||
if (!buffer->uncached)
|
||||
carveout_pages_zero(page, buffer->len, PAGE_KERNEL);
|
||||
else
|
||||
carveout_pages_zero(page, buffer->len, pgprot_writecombine(PAGE_KERNEL));
|
||||
}
|
||||
carveout_free(&sc_heap->carveout_heap, paddr, buffer->len);
|
||||
sg_free_table(table);
|
||||
kfree(buffer);
|
||||
}
|
||||
|
||||
static struct dma_heap_ops sc_heap_ops = {
|
||||
.allocate = sc_heap_allocate,
|
||||
};
|
||||
|
||||
int qcom_secure_carveout_heap_create(struct platform_heap *heap_data)
|
||||
{
|
||||
struct dma_heap_export_info exp_info;
|
||||
struct secure_carveout_heap *sc_heap;
|
||||
int ret;
|
||||
|
||||
sc_heap = kzalloc(sizeof(*sc_heap), GFP_KERNEL);
|
||||
if (!sc_heap)
|
||||
return -ENOMEM;
|
||||
|
||||
ret = __carveout_heap_init(heap_data, &sc_heap->carveout_heap, false);
|
||||
if (ret)
|
||||
goto err;
|
||||
|
||||
ret = hyp_assign_from_flags(heap_data->base, heap_data->size,
|
||||
heap_data->token);
|
||||
if (ret) {
|
||||
pr_err("secure_carveout_heap: Assign token 0x%x failed\n",
|
||||
heap_data->token);
|
||||
goto destroy_heap;
|
||||
}
|
||||
|
||||
sc_heap->token = heap_data->token;
|
||||
sc_heap->carveout_heap.is_secure = true;
|
||||
|
||||
exp_info.name = heap_data->name;
|
||||
exp_info.ops = &sc_heap_ops;
|
||||
exp_info.priv = sc_heap;
|
||||
|
||||
sc_heap->carveout_heap.heap = dma_heap_add(&exp_info);
|
||||
if (IS_ERR(sc_heap->carveout_heap.heap)) {
|
||||
ret = PTR_ERR(sc_heap->carveout_heap.heap);
|
||||
goto destroy_heap;
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
||||
destroy_heap:
|
||||
carveout_heap_destroy(&sc_heap->carveout_heap);
|
||||
err:
|
||||
kfree(sc_heap);
|
||||
|
||||
return ret;
|
||||
}
|
||||
25
drivers/dma-buf/heaps/qcom_carveout_heap.h
Normal file
25
drivers/dma-buf/heaps/qcom_carveout_heap.h
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifndef _QCOM_CARVEOUT_HEAP_H
|
||||
#define _QCOM_CARVEOUT_HEAP_H
|
||||
|
||||
#include "qcom_dt_parser.h"
|
||||
|
||||
#ifdef CONFIG_QCOM_DMABUF_HEAPS_CARVEOUT
|
||||
int qcom_secure_carveout_heap_create(struct platform_heap *heap_data);
|
||||
int qcom_carveout_heap_create(struct platform_heap *heap_data);
|
||||
#else
|
||||
static int qcom_secure_carveout_heap_create(struct platform_heap *heap_data)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
static int qcom_carveout_heap_create(struct platform_heap *heap_data)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* _QCOM_CARVEOUT_HEAP_H */
|
||||
198
drivers/dma-buf/heaps/qcom_cma_heap.c
Normal file
198
drivers/dma-buf/heaps/qcom_cma_heap.c
Normal file
|
|
@ -0,0 +1,198 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/*
|
||||
* DMABUF CMA heap exporter
|
||||
* Copied from drivers/dma-buf/heaps/cma_heap.c as of commit b61614ec318a
|
||||
* ("dma-buf: heaps: Add CMA heap to dmabuf heaps")
|
||||
*
|
||||
* Copyright (C) 2012, 2019 Linaro Ltd.
|
||||
* Author: <benjamin.gaignard@linaro.org> for ST-Ericsson.
|
||||
* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <linux/cma.h>
|
||||
#include <linux/device.h>
|
||||
#include <linux/dma-buf.h>
|
||||
#include <linux/dma-heap.h>
|
||||
#include <linux/dma-map-ops.h>
|
||||
#include <linux/err.h>
|
||||
#include <linux/errno.h>
|
||||
#include <linux/highmem.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/scatterlist.h>
|
||||
#include <linux/sched/signal.h>
|
||||
#include <linux/list.h>
|
||||
|
||||
#include "qcom_cma_heap.h"
|
||||
#include "qcom_sg_ops.h"
|
||||
|
||||
struct cma_heap {
|
||||
struct cma *cma;
|
||||
/* max_align is in units of page_order, similar to CONFIG_CMA_ALIGNMENT */
|
||||
u32 max_align;
|
||||
bool uncached;
|
||||
};
|
||||
|
||||
static void cma_heap_free(struct qcom_sg_buffer *buffer)
|
||||
{
|
||||
struct cma_heap *cma_heap;
|
||||
unsigned long nr_pages = buffer->len >> PAGE_SHIFT;
|
||||
struct page *cma_pages = sg_page(buffer->sg_table.sgl);
|
||||
|
||||
cma_heap = dma_heap_get_drvdata(buffer->heap);
|
||||
|
||||
/* free page list */
|
||||
sg_free_table(&buffer->sg_table);
|
||||
/* release memory */
|
||||
cma_release(cma_heap->cma, cma_pages, nr_pages);
|
||||
kfree(buffer);
|
||||
}
|
||||
|
||||
/* dmabuf heap CMA operations functions */
|
||||
struct dma_buf *cma_heap_allocate(struct dma_heap *heap,
|
||||
unsigned long len,
|
||||
unsigned long fd_flags,
|
||||
unsigned long heap_flags)
|
||||
{
|
||||
struct cma_heap *cma_heap;
|
||||
struct qcom_sg_buffer *helper_buffer;
|
||||
DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
|
||||
struct page *cma_pages;
|
||||
size_t size = PAGE_ALIGN(len);
|
||||
unsigned long nr_pages = size >> PAGE_SHIFT;
|
||||
unsigned long align = get_order(size);
|
||||
struct dma_buf *dmabuf;
|
||||
int ret = -ENOMEM;
|
||||
|
||||
cma_heap = dma_heap_get_drvdata(heap);
|
||||
|
||||
if (align > cma_heap->max_align)
|
||||
align = cma_heap->max_align;
|
||||
|
||||
helper_buffer = kzalloc(sizeof(*helper_buffer), GFP_KERNEL);
|
||||
if (!helper_buffer)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
helper_buffer->heap = heap;
|
||||
INIT_LIST_HEAD(&helper_buffer->attachments);
|
||||
mutex_init(&helper_buffer->lock);
|
||||
helper_buffer->len = size;
|
||||
helper_buffer->uncached = cma_heap->uncached;
|
||||
helper_buffer->free = cma_heap_free;
|
||||
|
||||
cma_pages = cma_alloc(cma_heap->cma, nr_pages, align, false);
|
||||
if (!cma_pages)
|
||||
goto free_buf;
|
||||
|
||||
if (PageHighMem(cma_pages)) {
|
||||
unsigned long nr_clear_pages = nr_pages;
|
||||
struct page *page = cma_pages;
|
||||
|
||||
while (nr_clear_pages > 0) {
|
||||
void *vaddr = kmap_atomic(page);
|
||||
|
||||
memset(vaddr, 0, PAGE_SIZE);
|
||||
kunmap_atomic(vaddr);
|
||||
/*
|
||||
* Avoid wasting time zeroing memory if the process
|
||||
* has been killed by SIGKILL
|
||||
*/
|
||||
if (fatal_signal_pending(current))
|
||||
goto free_cma;
|
||||
|
||||
page++;
|
||||
nr_clear_pages--;
|
||||
}
|
||||
} else {
|
||||
memset(page_address(cma_pages), 0, size);
|
||||
}
|
||||
|
||||
ret = sg_alloc_table(&helper_buffer->sg_table, 1, GFP_KERNEL);
|
||||
if (ret)
|
||||
goto free_cma;
|
||||
|
||||
sg_set_page(helper_buffer->sg_table.sgl, cma_pages, size, 0);
|
||||
|
||||
helper_buffer->vmperm = mem_buf_vmperm_alloc(&helper_buffer->sg_table);
|
||||
if (IS_ERR(helper_buffer->vmperm))
|
||||
goto free_sgtable;
|
||||
|
||||
if (helper_buffer->uncached) {
|
||||
dma_map_sgtable(dma_heap_get_dev(heap), &helper_buffer->sg_table,
|
||||
DMA_BIDIRECTIONAL, 0);
|
||||
dma_unmap_sgtable(dma_heap_get_dev(heap), &helper_buffer->sg_table,
|
||||
DMA_BIDIRECTIONAL, 0);
|
||||
}
|
||||
|
||||
/* create the dmabuf */
|
||||
exp_info.exp_name = dma_heap_get_name(heap);
|
||||
exp_info.size = helper_buffer->len;
|
||||
exp_info.flags = fd_flags;
|
||||
exp_info.priv = helper_buffer;
|
||||
dmabuf = mem_buf_dma_buf_export(&exp_info, &qcom_sg_buf_ops);
|
||||
if (IS_ERR(dmabuf)) {
|
||||
ret = PTR_ERR(dmabuf);
|
||||
goto vmperm_release;
|
||||
}
|
||||
|
||||
return dmabuf;
|
||||
|
||||
vmperm_release:
|
||||
mem_buf_vmperm_release(helper_buffer->vmperm);
|
||||
free_sgtable:
|
||||
sg_free_table(&helper_buffer->sg_table);
|
||||
free_cma:
|
||||
cma_release(cma_heap->cma, cma_pages, nr_pages);
|
||||
free_buf:
|
||||
kfree(helper_buffer);
|
||||
return ERR_PTR(ret);
|
||||
}
|
||||
|
||||
static const struct dma_heap_ops cma_heap_ops = {
|
||||
.allocate = cma_heap_allocate,
|
||||
};
|
||||
|
||||
static int __add_cma_heap(struct platform_heap *heap_data, void *data)
|
||||
{
|
||||
struct cma_heap *cma_heap;
|
||||
struct dma_heap_export_info exp_info;
|
||||
struct dma_heap *heap;
|
||||
|
||||
if (!heap_data->dev->cma_area) {
|
||||
pr_err("%s: CMA area for device uninitialized!\n", __func__);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
cma_heap = kzalloc(sizeof(*cma_heap), GFP_KERNEL);
|
||||
if (!cma_heap)
|
||||
return -ENOMEM;
|
||||
|
||||
cma_heap->cma = heap_data->dev->cma_area;
|
||||
cma_heap->max_align = CONFIG_CMA_ALIGNMENT;
|
||||
if (heap_data->max_align)
|
||||
cma_heap->max_align = heap_data->max_align;
|
||||
cma_heap->uncached = heap_data->is_uncached;
|
||||
|
||||
exp_info.name = heap_data->name;
|
||||
exp_info.ops = &cma_heap_ops;
|
||||
exp_info.priv = cma_heap;
|
||||
|
||||
heap = dma_heap_add(&exp_info);
|
||||
if (IS_ERR(heap)) {
|
||||
int ret = PTR_ERR(heap);
|
||||
|
||||
kfree(cma_heap);
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (cma_heap->uncached)
|
||||
dma_coerce_mask_and_coherent(dma_heap_get_dev(heap),
|
||||
DMA_BIT_MASK(64));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int qcom_add_cma_heap(struct platform_heap *heap_data)
|
||||
{
|
||||
return __add_cma_heap(heap_data, NULL);
|
||||
}
|
||||
20
drivers/dma-buf/heaps/qcom_cma_heap.h
Normal file
20
drivers/dma-buf/heaps/qcom_cma_heap.h
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifndef _QCOM_CMA_HEAP_H
|
||||
#define _QCOM_CMA_HEAP_H
|
||||
|
||||
#include "qcom_dt_parser.h"
|
||||
|
||||
#ifdef CONFIG_QCOM_DMABUF_HEAPS_CMA
|
||||
int qcom_add_cma_heap(struct platform_heap *heap_data);
|
||||
#else
|
||||
static int qcom_add_cma_heap(struct platform_heap *heap_data)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* _QCOM_CMA_HEAP_H */
|
||||
98
drivers/dma-buf/heaps/qcom_dma_heap.c
Normal file
98
drivers/dma-buf/heaps/qcom_dma_heap.c
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <linux/module.h>
|
||||
#include <linux/of_platform.h>
|
||||
#include <linux/platform_device.h>
|
||||
#include <linux/err.h>
|
||||
|
||||
#include <linux/qcom_dma_heap.h>
|
||||
#include "qcom_cma_heap.h"
|
||||
#include "qcom_dt_parser.h"
|
||||
#include "qcom_system_heap.h"
|
||||
#include "qcom_carveout_heap.h"
|
||||
#include "qcom_secure_system_heap.h"
|
||||
|
||||
static int qcom_dma_heap_probe(struct platform_device *pdev)
|
||||
{
|
||||
int ret = 0;
|
||||
int i;
|
||||
struct platform_data *heaps;
|
||||
|
||||
qcom_system_heap_create("qcom,system", "system", false);
|
||||
#ifdef CONFIG_QCOM_DMABUF_HEAPS_SYSTEM_UNCACHED
|
||||
qcom_system_heap_create("qcom,system-uncached", NULL, true);
|
||||
#endif
|
||||
qcom_secure_system_heap_create("qcom,secure-pixel", NULL,
|
||||
QCOM_DMA_HEAP_FLAG_CP_PIXEL);
|
||||
qcom_secure_system_heap_create("qcom,secure-non-pixel", NULL,
|
||||
QCOM_DMA_HEAP_FLAG_CP_NON_PIXEL);
|
||||
|
||||
heaps = parse_heap_dt(pdev);
|
||||
if (IS_ERR_OR_NULL(heaps))
|
||||
return PTR_ERR(heaps);
|
||||
|
||||
for (i = 0; i < heaps->nr; i++) {
|
||||
struct platform_heap *heap_data = &heaps->heaps[i];
|
||||
|
||||
switch (heap_data->type) {
|
||||
case HEAP_TYPE_SECURE_CARVEOUT:
|
||||
ret = qcom_secure_carveout_heap_create(heap_data);
|
||||
if (ret < 0)
|
||||
pr_err("%s: DMA-BUF Heap: Failed to create %s, error is %d\n",
|
||||
__func__, heap_data->name, ret);
|
||||
else if (!ret)
|
||||
pr_info("%s: DMA-BUF Heap: Created %s\n", __func__,
|
||||
heap_data->name);
|
||||
break;
|
||||
case HEAP_TYPE_CARVEOUT:
|
||||
ret = qcom_carveout_heap_create(heap_data);
|
||||
if (ret < 0)
|
||||
pr_err("%s: DMA-BUF Heap: Failed to create %s, error is %d\n",
|
||||
__func__, heap_data->name, ret);
|
||||
else if (!ret)
|
||||
pr_info("%s: DMA-BUF Heap: Created %s\n", __func__,
|
||||
heap_data->name);
|
||||
break;
|
||||
case HEAP_TYPE_CMA:
|
||||
ret = qcom_add_cma_heap(heap_data);
|
||||
if (ret < 0)
|
||||
pr_err("%s: DMA-BUF Heap: Failed to create %s, error is %d\n",
|
||||
__func__, heap_data->name, ret);
|
||||
else if (!ret)
|
||||
pr_info("%s: DMA-BUF Heap: Created %s\n", __func__,
|
||||
heap_data->name);
|
||||
break;
|
||||
default:
|
||||
pr_err("%s: Unknown heap type %u\n", __func__, heap_data->type);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
free_pdata(heaps);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static const struct of_device_id qcom_dma_heap_match_table[] = {
|
||||
{.compatible = "qcom,dma-heaps"},
|
||||
{},
|
||||
};
|
||||
|
||||
static struct platform_driver qcom_dma_heap_driver = {
|
||||
.probe = qcom_dma_heap_probe,
|
||||
.driver = {
|
||||
.name = "qcom-dma-heap",
|
||||
.of_match_table = qcom_dma_heap_match_table,
|
||||
},
|
||||
};
|
||||
|
||||
static int __init init_heap_driver(void)
|
||||
{
|
||||
return platform_driver_register(&qcom_dma_heap_driver);
|
||||
}
|
||||
module_init(init_heap_driver);
|
||||
|
||||
MODULE_LICENSE("GPL v2");
|
||||
45
drivers/dma-buf/heaps/qcom_dma_heap_priv.h
Normal file
45
drivers/dma-buf/heaps/qcom_dma_heap_priv.h
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (C) 2011 Google, Inc.
|
||||
* Copyright (C) 2019 Linaro Ltd.
|
||||
* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifndef _QCOM_DMA_HEAP_PRIV_H
|
||||
#define _QCOM_DMA_HEAP_PRIV_H
|
||||
|
||||
#include <linux/dma-heap.h>
|
||||
#include <linux/list.h>
|
||||
|
||||
/**
|
||||
* struct heap_helper_buffer - helper buffer metadata
|
||||
* @heap: back pointer to the heap the buffer came from
|
||||
* @dmabuf: backing dma-buf for this buffer
|
||||
* @size: size of the buffer
|
||||
* @priv_virt pointer to heap specific private value
|
||||
* @lock mutext to protect the data in this structure
|
||||
* @vmap_cnt count of vmap references on the buffer
|
||||
* @vaddr vmap'ed virtual address
|
||||
* @pagecount number of pages in the buffer
|
||||
* @pages list of page pointers
|
||||
* @attachments list of device attachments
|
||||
*
|
||||
* @free heap callback to free the buffer
|
||||
*/
|
||||
struct heap_helper_buffer {
|
||||
struct dma_heap *heap;
|
||||
struct dma_buf *dmabuf;
|
||||
size_t size;
|
||||
|
||||
void *priv_virt;
|
||||
struct mutex lock;
|
||||
int vmap_cnt;
|
||||
void *vaddr;
|
||||
pgoff_t pagecount;
|
||||
struct page **pages;
|
||||
struct list_head attachments;
|
||||
|
||||
void (*free)(struct heap_helper_buffer *buffer);
|
||||
};
|
||||
|
||||
#endif /* _QCOM_DMA_HEAP_PRIV_H */
|
||||
272
drivers/dma-buf/heaps/qcom_dma_heap_secure_utils.c
Normal file
272
drivers/dma-buf/heaps/qcom_dma_heap_secure_utils.c
Normal file
|
|
@ -0,0 +1,272 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <linux/slab.h>
|
||||
#include <soc/qcom/secure_buffer.h>
|
||||
#include <linux/qcom_dma_heap.h>
|
||||
|
||||
int get_secure_vmid(unsigned long flags)
|
||||
{
|
||||
if (flags & QCOM_DMA_HEAP_FLAG_CP_TZ)
|
||||
return VMID_TZ;
|
||||
if (flags & QCOM_DMA_HEAP_FLAG_CP_TOUCH)
|
||||
return VMID_CP_TOUCH;
|
||||
if (flags & QCOM_DMA_HEAP_FLAG_CP_BITSTREAM)
|
||||
return VMID_CP_BITSTREAM;
|
||||
if (flags & QCOM_DMA_HEAP_FLAG_CP_PIXEL)
|
||||
return VMID_CP_PIXEL;
|
||||
if (flags & QCOM_DMA_HEAP_FLAG_CP_NON_PIXEL)
|
||||
return VMID_CP_NON_PIXEL;
|
||||
if (flags & QCOM_DMA_HEAP_FLAG_CP_CAMERA)
|
||||
return VMID_CP_CAMERA;
|
||||
if (flags & QCOM_DMA_HEAP_FLAG_CP_SEC_DISPLAY)
|
||||
return VMID_CP_SEC_DISPLAY;
|
||||
if (flags & QCOM_DMA_HEAP_FLAG_CP_APP)
|
||||
return VMID_CP_APP;
|
||||
if (flags & QCOM_DMA_HEAP_FLAG_CP_CAMERA_PREVIEW)
|
||||
return VMID_CP_CAMERA_PREVIEW;
|
||||
if (flags & QCOM_DMA_HEAP_FLAG_CP_SPSS_SP)
|
||||
return VMID_CP_SPSS_SP;
|
||||
if (flags & QCOM_DMA_HEAP_FLAG_CP_SPSS_SP_SHARED)
|
||||
return VMID_CP_SPSS_SP_SHARED;
|
||||
if (flags & QCOM_DMA_HEAP_FLAG_CP_SPSS_HLOS_SHARED)
|
||||
return VMID_CP_SPSS_HLOS_SHARED;
|
||||
if (flags & QCOM_DMA_HEAP_FLAG_CP_CDSP)
|
||||
return VMID_CP_CDSP;
|
||||
if (flags & QCOM_DMA_HEAP_FLAG_CP_MSS_MSA)
|
||||
return VMID_MSS_MSA;
|
||||
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
static unsigned int count_set_bits(unsigned long val)
|
||||
{
|
||||
return ((unsigned int)bitmap_weight(&val, BITS_PER_LONG));
|
||||
}
|
||||
|
||||
bool qcom_is_buffer_hlos_accessible(unsigned long flags)
|
||||
{
|
||||
if (!(flags & QCOM_DMA_HEAP_FLAG_CP_HLOS) &&
|
||||
!(flags & QCOM_DMA_HEAP_FLAG_CP_SPSS_HLOS_SHARED))
|
||||
return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static int get_vmid(unsigned long flags)
|
||||
{
|
||||
int vmid;
|
||||
|
||||
vmid = get_secure_vmid(flags);
|
||||
if (vmid < 0) {
|
||||
if (flags & QCOM_DMA_HEAP_FLAG_CP_HLOS)
|
||||
vmid = VMID_HLOS;
|
||||
}
|
||||
return vmid;
|
||||
}
|
||||
|
||||
static int populate_vm_list(unsigned long flags, unsigned int *vm_list,
|
||||
int nelems)
|
||||
{
|
||||
unsigned int itr = 0;
|
||||
int vmid;
|
||||
|
||||
flags = flags & QCOM_DMA_HEAP_FLAGS_CP_MASK;
|
||||
if (!flags)
|
||||
return -EINVAL;
|
||||
|
||||
for_each_set_bit(itr, &flags, BITS_PER_LONG) {
|
||||
vmid = get_vmid(0x1UL << itr);
|
||||
if (vmid < 0 || !nelems)
|
||||
return -EINVAL;
|
||||
|
||||
vm_list[nelems - 1] = vmid;
|
||||
nelems--;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int hyp_unassign_sg(struct sg_table *sgt, int *source_vm_list,
|
||||
int source_nelems, bool clear_page_private)
|
||||
{
|
||||
u32 dest_vmid = VMID_HLOS;
|
||||
u32 dest_perms = PERM_READ | PERM_WRITE | PERM_EXEC;
|
||||
struct scatterlist *sg;
|
||||
int ret, i;
|
||||
|
||||
if (source_nelems <= 0) {
|
||||
pr_err("%s: source_nelems invalid\n",
|
||||
__func__);
|
||||
ret = -EINVAL;
|
||||
goto out;
|
||||
}
|
||||
|
||||
ret = hyp_assign_table(sgt, source_vm_list, source_nelems, &dest_vmid,
|
||||
&dest_perms, 1);
|
||||
if (ret)
|
||||
goto out;
|
||||
|
||||
if (clear_page_private)
|
||||
for_each_sg(sgt->sgl, sg, sgt->nents, i)
|
||||
ClearPagePrivate(sg_page(sg));
|
||||
out:
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int hyp_assign_sg(struct sg_table *sgt, int *dest_vm_list,
|
||||
int dest_nelems, bool set_page_private)
|
||||
{
|
||||
u32 source_vmid = VMID_HLOS;
|
||||
struct scatterlist *sg;
|
||||
int *dest_perms;
|
||||
int ret, i;
|
||||
|
||||
if (dest_nelems <= 0) {
|
||||
pr_err("%s: dest_nelems invalid\n",
|
||||
__func__);
|
||||
ret = -EINVAL;
|
||||
goto out;
|
||||
}
|
||||
|
||||
dest_perms = kcalloc(dest_nelems, sizeof(*dest_perms), GFP_KERNEL);
|
||||
if (!dest_perms) {
|
||||
ret = -ENOMEM;
|
||||
goto out;
|
||||
}
|
||||
|
||||
for (i = 0; i < dest_nelems; i++)
|
||||
dest_perms[i] = msm_secure_get_vmid_perms(dest_vm_list[i]);
|
||||
|
||||
ret = hyp_assign_table(sgt, &source_vmid, 1,
|
||||
dest_vm_list, dest_perms, dest_nelems);
|
||||
|
||||
if (ret) {
|
||||
pr_err("%s: Assign call failed\n",
|
||||
__func__);
|
||||
goto out_free_dest;
|
||||
}
|
||||
if (set_page_private)
|
||||
for_each_sg(sgt->sgl, sg, sgt->nents, i)
|
||||
SetPagePrivate(sg_page(sg));
|
||||
|
||||
out_free_dest:
|
||||
kfree(dest_perms);
|
||||
out:
|
||||
return ret;
|
||||
}
|
||||
|
||||
int hyp_unassign_sg_from_flags(struct sg_table *sgt, unsigned long flags,
|
||||
bool set_page_private)
|
||||
{
|
||||
int ret;
|
||||
int *source_vm_list;
|
||||
int source_nelems;
|
||||
|
||||
source_nelems = count_set_bits(flags & QCOM_DMA_HEAP_FLAGS_CP_MASK);
|
||||
source_vm_list = kcalloc(source_nelems, sizeof(*source_vm_list),
|
||||
GFP_KERNEL);
|
||||
if (!source_vm_list)
|
||||
return -ENOMEM;
|
||||
ret = populate_vm_list(flags, source_vm_list, source_nelems);
|
||||
if (ret) {
|
||||
pr_err("%s: Failed to get secure vmids\n", __func__);
|
||||
goto out_free_source;
|
||||
}
|
||||
|
||||
ret = hyp_unassign_sg(sgt, source_vm_list, source_nelems,
|
||||
set_page_private);
|
||||
|
||||
out_free_source:
|
||||
kfree(source_vm_list);
|
||||
return ret;
|
||||
}
|
||||
|
||||
int hyp_assign_sg_from_flags(struct sg_table *sgt, unsigned long flags,
|
||||
bool set_page_private)
|
||||
{
|
||||
int ret;
|
||||
int *dest_vm_list = NULL;
|
||||
int dest_nelems;
|
||||
|
||||
dest_nelems = count_set_bits(flags & QCOM_DMA_HEAP_FLAGS_CP_MASK);
|
||||
dest_vm_list = kcalloc(dest_nelems, sizeof(*dest_vm_list), GFP_KERNEL);
|
||||
if (!dest_vm_list) {
|
||||
ret = -ENOMEM;
|
||||
goto out;
|
||||
}
|
||||
|
||||
ret = populate_vm_list(flags, dest_vm_list, dest_nelems);
|
||||
if (ret) {
|
||||
pr_err("%s: Failed to get secure vmid(s)\n", __func__);
|
||||
goto out_free_dest_vm;
|
||||
}
|
||||
|
||||
ret = hyp_assign_sg(sgt, dest_vm_list, dest_nelems,
|
||||
set_page_private);
|
||||
|
||||
out_free_dest_vm:
|
||||
kfree(dest_vm_list);
|
||||
out:
|
||||
return ret;
|
||||
}
|
||||
|
||||
int get_vmperm_from_ion_flags(unsigned long flags, int **_vmids,
|
||||
int **_perms, u32 *_nr)
|
||||
{
|
||||
int *vmids, *modes;
|
||||
u32 nr, i;
|
||||
|
||||
if (!_vmids || !_perms || !_nr)
|
||||
return -EINVAL;
|
||||
|
||||
nr = count_set_bits(flags);
|
||||
vmids = kcalloc(nr, sizeof(*vmids), GFP_KERNEL);
|
||||
if (!vmids)
|
||||
return -ENOMEM;
|
||||
|
||||
modes = kcalloc(nr, sizeof(*modes), GFP_KERNEL);
|
||||
if (!modes) {
|
||||
kfree(vmids);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
if ((flags & ~QCOM_DMA_HEAP_FLAGS_CP_MASK) ||
|
||||
populate_vm_list(flags, vmids, nr)) {
|
||||
pr_err("%s: Failed to parse secure flags 0x%lx\n", __func__,
|
||||
flags);
|
||||
kfree(modes);
|
||||
kfree(vmids);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
for (i = 0; i < nr; i++)
|
||||
modes[i] = msm_secure_get_vmid_perms(vmids[i]);
|
||||
|
||||
*_vmids = vmids;
|
||||
*_perms = modes;
|
||||
*_nr = nr;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int hyp_assign_from_flags(u64 base, u64 size, unsigned long flags)
|
||||
{
|
||||
int *vmids, *modes;
|
||||
u32 nr;
|
||||
u32 src_vm = VMID_HLOS;
|
||||
int ret;
|
||||
|
||||
ret = get_vmperm_from_ion_flags(flags, &vmids, &modes, &nr);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
ret = hyp_assign_phys(base, size, &src_vm, 1, vmids, modes, nr);
|
||||
if (ret)
|
||||
pr_err("%s: Assign call failed, flags 0x%lx\n", __func__,
|
||||
flags);
|
||||
|
||||
kfree(modes);
|
||||
kfree(vmids);
|
||||
return ret;
|
||||
}
|
||||
24
drivers/dma-buf/heaps/qcom_dma_heap_secure_utils.h
Normal file
24
drivers/dma-buf/heaps/qcom_dma_heap_secure_utils.h
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifndef _QCOM_DMA_HEAP_SECURE_UTILS_H
|
||||
#define _QCOM_DMA_HEAP_SECURE_UTILS_H
|
||||
|
||||
int hyp_assign_sg_from_flags(struct sg_table *sgt, unsigned long flags,
|
||||
bool set_page_private);
|
||||
|
||||
int hyp_unassign_sg_from_flags(struct sg_table *sgt, unsigned long flags,
|
||||
bool set_page_private);
|
||||
|
||||
/*
|
||||
* This function handles conversion from the legacy ion flag based interface
|
||||
* to a precise list of vmids and permissions.
|
||||
*/
|
||||
int get_vmperm_from_ion_flags(unsigned long flags, int **vmids, int **perms, u32 *nr);
|
||||
int hyp_assign_from_flags(u64 base, u64 size, unsigned long flags);
|
||||
bool qcom_is_buffer_hlos_accessible(unsigned long flags);
|
||||
int get_secure_vmid(unsigned long flags);
|
||||
|
||||
#endif /* _QCOM_DMA_HEAP_SECURE_UTILS_H */
|
||||
203
drivers/dma-buf/heaps/qcom_dt_parser.c
Normal file
203
drivers/dma-buf/heaps/qcom_dt_parser.c
Normal file
|
|
@ -0,0 +1,203 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <linux/of.h>
|
||||
#include <linux/platform_device.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/of_platform.h>
|
||||
#include <linux/of_address.h>
|
||||
#include <linux/of_reserved_mem.h>
|
||||
|
||||
#include <linux/qcom_dma_heap.h>
|
||||
#include "qcom_dt_parser.h"
|
||||
|
||||
#ifdef CONFIG_PANIC_ON_QCOM_DMA_HEAPS_FAILURE
|
||||
#define QCOM_DMA_HEAP_WARN(fmt...) panic(fmt)
|
||||
#else /* CONFIG_PANIC_ON_QCOM_DMA_HEAPS_FAILURE */
|
||||
#define QCOM_DMA_HEAP_WARN(fmt...) WARN(1, fmt)
|
||||
#endif /* CONFIG_PANIC_ON_QCOM_DMA_HEAPS_FAILURE */
|
||||
|
||||
static int populate_heap(struct device_node *node,
|
||||
struct platform_heap *heap)
|
||||
{
|
||||
int ret;
|
||||
|
||||
/* Mandatory properties */
|
||||
ret = of_property_read_string(node, "qcom,dma-heap-name", &heap->name);
|
||||
if (ret)
|
||||
goto err;
|
||||
|
||||
ret = of_property_read_u32(node, "qcom,dma-heap-type", &heap->type);
|
||||
if (ret) {
|
||||
pr_err("Reading %s property in node %s failed with err %d.\n",
|
||||
"qcom,dma-heap-type", of_node_full_name(node), ret);
|
||||
goto err;
|
||||
}
|
||||
|
||||
/* Optional properties */
|
||||
heap->is_dynamic = of_property_read_bool(node, "qcom,dynamic-heap");
|
||||
|
||||
heap->is_uncached = of_property_read_bool(node, "qcom,uncached-heap");
|
||||
|
||||
ret = of_property_read_u32(node, "qcom,token", &heap->token);
|
||||
if (ret && ret != -EINVAL)
|
||||
goto err;
|
||||
|
||||
ret = of_property_read_u32(node, "qcom,max-align", &heap->max_align);
|
||||
if (ret && ret != -EINVAL)
|
||||
goto err;
|
||||
|
||||
return 0;
|
||||
|
||||
err:
|
||||
if (ret)
|
||||
QCOM_DMA_HEAP_WARN("%s: Unable to populate heap %s, err: %d\n",
|
||||
__func__, of_node_full_name(node), ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
void free_pdata(const struct platform_data *pdata)
|
||||
{
|
||||
kfree(pdata->heaps);
|
||||
kfree(pdata);
|
||||
}
|
||||
|
||||
static int heap_dt_init(struct device_node *mem_node,
|
||||
struct platform_heap *heap)
|
||||
{
|
||||
const __be32 *basep;
|
||||
u64 base, size;
|
||||
struct device *dev = heap->dev;
|
||||
struct reserved_mem *rmem;
|
||||
int ret = 0;
|
||||
|
||||
|
||||
rmem = of_reserved_mem_lookup(mem_node);
|
||||
|
||||
if (!rmem) {
|
||||
dev_err(dev, "Failed to find reserved memory region\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
/*
|
||||
* We only need to call this when the memory-region is managed by
|
||||
* a reserved memory region driver (e.g. CMA, coherent, etc). In that
|
||||
* case, they will have ops for device specific initialization for
|
||||
* the memory region. Otherwise, we have a pure carveout, which needs
|
||||
* not be initialized.
|
||||
*/
|
||||
if (rmem->ops) {
|
||||
ret = of_reserved_mem_device_init_by_idx(dev, dev->of_node, 0);
|
||||
if (ret) {
|
||||
dev_err(dev,
|
||||
"Failed to initialize memory region rc: %d\n",
|
||||
ret);
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
basep = of_get_address(mem_node, 0, &size, NULL);
|
||||
if (basep) {
|
||||
base = of_translate_address(mem_node, basep);
|
||||
if (base != OF_BAD_ADDR) {
|
||||
heap->base = base;
|
||||
heap->size = size;
|
||||
} else {
|
||||
ret = -EINVAL;
|
||||
dev_err(heap->dev,
|
||||
"Failed to get heap base/size\n");
|
||||
of_reserved_mem_device_release(dev);
|
||||
}
|
||||
}
|
||||
heap->is_nomap = of_property_read_bool(mem_node, "no-map");
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void release_reserved_memory_regions(struct platform_heap *heaps,
|
||||
int idx)
|
||||
{
|
||||
struct device *dev;
|
||||
struct device_node *node, *mem_node;
|
||||
|
||||
for (idx = idx - 1; idx >= 0; idx--) {
|
||||
dev = heaps[idx].dev;
|
||||
node = dev->of_node;
|
||||
mem_node = of_parse_phandle(node, "memory-region", 0);
|
||||
|
||||
if (mem_node)
|
||||
of_reserved_mem_device_release(dev);
|
||||
|
||||
of_node_put(mem_node);
|
||||
}
|
||||
}
|
||||
|
||||
struct platform_data *parse_heap_dt(struct platform_device *pdev)
|
||||
{
|
||||
struct platform_data *pdata = NULL;
|
||||
struct device_node *node;
|
||||
struct device_node *mem_node;
|
||||
struct platform_device *new_dev = NULL;
|
||||
const struct device_node *dt_node = pdev->dev.of_node;
|
||||
int ret;
|
||||
int idx = 0;
|
||||
|
||||
pdata = kzalloc(sizeof(*pdata), GFP_KERNEL);
|
||||
if (!pdata)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
for_each_available_child_of_node(dt_node, node)
|
||||
pdata->nr++;
|
||||
|
||||
/*
|
||||
* No heaps defined in the devicetree. However, there may be other
|
||||
* heaps (e.g. system heaps) that do not need to be defined in the
|
||||
* devicetree.
|
||||
*/
|
||||
if (!pdata->nr)
|
||||
goto out;
|
||||
|
||||
pdata->heaps = kcalloc(pdata->nr, sizeof(*pdata->heaps), GFP_KERNEL);
|
||||
if (!pdata->heaps) {
|
||||
kfree(pdata);
|
||||
return ERR_PTR(-ENOMEM);
|
||||
}
|
||||
|
||||
for_each_available_child_of_node(dt_node, node) {
|
||||
new_dev = of_platform_device_create(node, NULL, &pdev->dev);
|
||||
if (!new_dev) {
|
||||
pr_err("Failed to create device %s\n", node->name);
|
||||
ret = -EINVAL;
|
||||
goto free_heaps;
|
||||
}
|
||||
of_dma_configure(&new_dev->dev, node, true);
|
||||
|
||||
pdata->heaps[idx].dev = &new_dev->dev;
|
||||
|
||||
ret = populate_heap(node, &pdata->heaps[idx]);
|
||||
if (ret)
|
||||
goto free_heaps;
|
||||
|
||||
mem_node = of_parse_phandle(node, "memory-region", 0);
|
||||
if (mem_node) {
|
||||
ret = heap_dt_init(mem_node, &pdata->heaps[idx]);
|
||||
if (ret)
|
||||
goto free_heaps;
|
||||
|
||||
of_node_put(mem_node);
|
||||
}
|
||||
|
||||
++idx;
|
||||
}
|
||||
|
||||
out:
|
||||
return pdata;
|
||||
|
||||
free_heaps:
|
||||
of_node_put(node);
|
||||
release_reserved_memory_regions(pdata->heaps, idx);
|
||||
free_pdata(pdata);
|
||||
return ERR_PTR(ret);
|
||||
}
|
||||
57
drivers/dma-buf/heaps/qcom_dt_parser.h
Normal file
57
drivers/dma-buf/heaps/qcom_dt_parser.h
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifndef _QCOM_DT_PARSER_H
|
||||
#define _QCOM_DT_PARSER_H
|
||||
|
||||
#include <linux/device.h>
|
||||
#include <linux/platform_device.h>
|
||||
|
||||
/**
|
||||
* struct platform_heap - defines a heap in the given platform
|
||||
* @type: type of the heap
|
||||
* @name: used for debug purposes
|
||||
* @base: base address of heap in physical memory if applicable
|
||||
* @size: size of the heap in bytes if applicable
|
||||
* @dev: the device associated with the heap's DT node
|
||||
* @is_dynamic: indicates if memory can be added or removed from carveout heaps
|
||||
* is_nomap: indicates if this heap is assosicated with a memory region which is
|
||||
* which has no-map property.
|
||||
* is_uncached: indicates if this heap provides uncached memory or not
|
||||
* @token: the end points to which memory for secure carveout memory is
|
||||
* assigned to
|
||||
* @max_align: page order of the maximum alignment. Used by cma heap.
|
||||
*
|
||||
* Provided by the board file.
|
||||
*/
|
||||
struct platform_heap {
|
||||
u32 type;
|
||||
const char *name;
|
||||
phys_addr_t base;
|
||||
size_t size;
|
||||
struct device *dev;
|
||||
bool is_uncached;
|
||||
bool is_dynamic;
|
||||
bool is_nomap;
|
||||
u32 token;
|
||||
u32 max_align;
|
||||
};
|
||||
|
||||
/**
|
||||
* struct platform_data - array of platform heaps passed from board file
|
||||
* @nr: number of structures in the array
|
||||
* @heaps: array of platform_heap structions
|
||||
*
|
||||
* Provided by the board file in the form of platform data to a platform device.
|
||||
*/
|
||||
struct platform_data {
|
||||
int nr;
|
||||
struct platform_heap *heaps;
|
||||
};
|
||||
|
||||
struct platform_data *parse_heap_dt(struct platform_device *pdev);
|
||||
void free_pdata(const struct platform_data *pdata);
|
||||
|
||||
#endif /* _QCOM_DT_PARSER_H */
|
||||
332
drivers/dma-buf/heaps/qcom_dynamic_page_pool.c
Normal file
332
drivers/dma-buf/heaps/qcom_dynamic_page_pool.c
Normal file
|
|
@ -0,0 +1,332 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/*
|
||||
* Dynamic page pool system
|
||||
*
|
||||
* Taken from:
|
||||
* mm/dynamic_page_pool.c
|
||||
* Git-repo: https://git.linaro.org/people/john.stultz/android-dev.git
|
||||
* Branch: dma-buf-heap-perf
|
||||
* Git-commit: 458ea8030852755867bdc0384aa40f97aba7a572
|
||||
*
|
||||
* Based on the ION page pool code
|
||||
* Copyright (C) 2011 Google, Inc.
|
||||
*
|
||||
* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <linux/list.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/swap.h>
|
||||
#include <linux/sched/signal.h>
|
||||
#include <uapi/linux/sched/types.h>
|
||||
|
||||
#include "qcom_dynamic_page_pool.h"
|
||||
|
||||
static LIST_HEAD(pool_list);
|
||||
static DEFINE_MUTEX(pool_list_lock);
|
||||
|
||||
void dynamic_page_pool_add(struct dynamic_page_pool *pool, struct page *page)
|
||||
{
|
||||
mutex_lock(&pool->mutex);
|
||||
if (PageHighMem(page)) {
|
||||
list_add_tail(&page->lru, &pool->high_items);
|
||||
pool->high_count++;
|
||||
} else {
|
||||
list_add_tail(&page->lru, &pool->low_items);
|
||||
pool->low_count++;
|
||||
}
|
||||
|
||||
atomic_inc(&pool->count);
|
||||
mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE,
|
||||
1 << pool->order);
|
||||
mutex_unlock(&pool->mutex);
|
||||
}
|
||||
|
||||
struct page *dynamic_page_pool_remove(struct dynamic_page_pool *pool, bool high)
|
||||
{
|
||||
struct page *page;
|
||||
|
||||
if (high) {
|
||||
BUG_ON(!pool->high_count);
|
||||
page = list_first_entry(&pool->high_items, struct page, lru);
|
||||
pool->high_count--;
|
||||
} else {
|
||||
BUG_ON(!pool->low_count);
|
||||
page = list_first_entry(&pool->low_items, struct page, lru);
|
||||
pool->low_count--;
|
||||
}
|
||||
|
||||
atomic_dec(&pool->count);
|
||||
list_del(&page->lru);
|
||||
mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE,
|
||||
-(1 << pool->order));
|
||||
return page;
|
||||
}
|
||||
|
||||
void dynamic_page_pool_free(struct dynamic_page_pool *pool, struct page *page)
|
||||
{
|
||||
BUG_ON(pool->order != compound_order(page));
|
||||
|
||||
dynamic_page_pool_add(pool, page);
|
||||
}
|
||||
|
||||
int dynamic_page_pool_total(struct dynamic_page_pool *pool, bool high)
|
||||
{
|
||||
int count = pool->low_count;
|
||||
|
||||
if (high)
|
||||
count += pool->high_count;
|
||||
|
||||
return count << pool->order;
|
||||
}
|
||||
|
||||
struct dynamic_page_pool *dynamic_page_pool_create(gfp_t gfp_mask, unsigned int order)
|
||||
{
|
||||
struct dynamic_page_pool *pool = kmalloc(sizeof(*pool), GFP_KERNEL);
|
||||
|
||||
if (!pool)
|
||||
return NULL;
|
||||
pool->high_count = 0;
|
||||
pool->low_count = 0;
|
||||
INIT_LIST_HEAD(&pool->low_items);
|
||||
INIT_LIST_HEAD(&pool->high_items);
|
||||
pool->gfp_mask = gfp_mask | __GFP_COMP;
|
||||
pool->order = order;
|
||||
mutex_init(&pool->mutex);
|
||||
|
||||
mutex_lock(&pool_list_lock);
|
||||
list_add(&pool->list, &pool_list);
|
||||
mutex_unlock(&pool_list_lock);
|
||||
|
||||
return pool;
|
||||
}
|
||||
|
||||
void dynamic_page_pool_destroy(struct dynamic_page_pool *pool)
|
||||
{
|
||||
struct page *page, *tmp;
|
||||
LIST_HEAD(pages);
|
||||
int num_pages = 0;
|
||||
int ret = DYNAMIC_POOL_SUCCESS;
|
||||
|
||||
/* Remove us from the pool list */
|
||||
mutex_lock(&pool_list_lock);
|
||||
list_del(&pool->list);
|
||||
mutex_unlock(&pool_list_lock);
|
||||
|
||||
/* Free any remaining pages in the pool */
|
||||
mutex_lock(&pool->mutex);
|
||||
while (true) {
|
||||
if (pool->low_count)
|
||||
page = dynamic_page_pool_remove(pool, false);
|
||||
else if (pool->high_count)
|
||||
page = dynamic_page_pool_remove(pool, true);
|
||||
else
|
||||
break;
|
||||
|
||||
list_add(&page->lru, &pages);
|
||||
num_pages++;
|
||||
}
|
||||
mutex_unlock(&pool->mutex);
|
||||
|
||||
if (num_pages && pool->prerelease_callback)
|
||||
ret = pool->prerelease_callback(pool, &pages, num_pages);
|
||||
|
||||
if (ret != DYNAMIC_POOL_SUCCESS) {
|
||||
pr_err("Failed to reclaim pages when destroying the pool!\n");
|
||||
return;
|
||||
}
|
||||
|
||||
list_for_each_entry_safe(page, tmp, &pages, lru) {
|
||||
list_del(&page->lru);
|
||||
__free_pages(page, pool->order);
|
||||
}
|
||||
|
||||
kfree(pool);
|
||||
}
|
||||
|
||||
static int dynamic_page_pool_do_shrink(struct dynamic_page_pool *pool, gfp_t gfp_mask,
|
||||
int nr_to_scan)
|
||||
{
|
||||
int freed = 0;
|
||||
bool high;
|
||||
struct page *page, *tmp;
|
||||
LIST_HEAD(pages);
|
||||
int ret = DYNAMIC_POOL_SUCCESS;
|
||||
|
||||
if (current_is_kswapd())
|
||||
high = true;
|
||||
else
|
||||
high = !!(gfp_mask & __GFP_HIGHMEM);
|
||||
|
||||
if (nr_to_scan == 0)
|
||||
return dynamic_page_pool_total(pool, high);
|
||||
|
||||
while (freed < nr_to_scan) {
|
||||
mutex_lock(&pool->mutex);
|
||||
if (pool->low_count) {
|
||||
page = dynamic_page_pool_remove(pool, false);
|
||||
} else if (high && pool->high_count) {
|
||||
page = dynamic_page_pool_remove(pool, true);
|
||||
} else {
|
||||
mutex_unlock(&pool->mutex);
|
||||
break;
|
||||
}
|
||||
mutex_unlock(&pool->mutex);
|
||||
list_add(&page->lru, &pages);
|
||||
freed += (1 << pool->order);
|
||||
}
|
||||
|
||||
if (freed && pool->prerelease_callback)
|
||||
ret = pool->prerelease_callback(pool, &pages, freed >> pool->order);
|
||||
|
||||
if (ret != DYNAMIC_POOL_SUCCESS) {
|
||||
pr_err("Failed to reclaim secure page pool pages!\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
list_for_each_entry_safe(page, tmp, &pages, lru) {
|
||||
list_del(&page->lru);
|
||||
__free_pages(page, pool->order);
|
||||
}
|
||||
|
||||
return freed;
|
||||
}
|
||||
|
||||
void dynamic_page_pool_shrink_high_and_low(struct dynamic_page_pool **pools_list,
|
||||
int num_pools, int nr_to_scan)
|
||||
{
|
||||
int i, remaining = nr_to_scan;
|
||||
|
||||
mutex_lock(&pool_list_lock);
|
||||
for (i = 0; i < num_pools; i++) {
|
||||
remaining -= dynamic_page_pool_do_shrink(pools_list[i], __GFP_HIGHMEM,
|
||||
remaining);
|
||||
|
||||
if (remaining <= 0) {
|
||||
mutex_unlock(&pool_list_lock);
|
||||
return;
|
||||
}
|
||||
}
|
||||
mutex_unlock(&pool_list_lock);
|
||||
}
|
||||
|
||||
static int dynamic_page_pool_shrink(gfp_t gfp_mask, int nr_to_scan)
|
||||
{
|
||||
struct dynamic_page_pool *pool;
|
||||
int nr_total = 0;
|
||||
int nr_freed;
|
||||
int only_scan = 0;
|
||||
|
||||
if (!nr_to_scan)
|
||||
only_scan = 1;
|
||||
|
||||
mutex_lock(&pool_list_lock);
|
||||
list_for_each_entry(pool, &pool_list, list) {
|
||||
if (only_scan) {
|
||||
nr_total += dynamic_page_pool_do_shrink(pool,
|
||||
gfp_mask,
|
||||
nr_to_scan);
|
||||
} else {
|
||||
nr_freed = dynamic_page_pool_do_shrink(pool,
|
||||
gfp_mask,
|
||||
nr_to_scan);
|
||||
nr_to_scan -= nr_freed;
|
||||
nr_total += nr_freed;
|
||||
if (nr_to_scan <= 0)
|
||||
break;
|
||||
}
|
||||
}
|
||||
mutex_unlock(&pool_list_lock);
|
||||
|
||||
return nr_total;
|
||||
}
|
||||
|
||||
static unsigned long dynamic_page_pool_shrink_count(struct shrinker *shrinker,
|
||||
struct shrink_control *sc)
|
||||
{
|
||||
return dynamic_page_pool_shrink(sc->gfp_mask, 0);
|
||||
}
|
||||
|
||||
static unsigned long dynamic_page_pool_shrink_scan(struct shrinker *shrinker,
|
||||
struct shrink_control *sc)
|
||||
{
|
||||
int to_scan = sc->nr_to_scan;
|
||||
|
||||
if (to_scan == 0)
|
||||
return 0;
|
||||
|
||||
return dynamic_page_pool_shrink(sc->gfp_mask, to_scan);
|
||||
}
|
||||
|
||||
struct dynamic_page_pool **dynamic_page_pool_create_pools(int vmid,
|
||||
prerelease_callback callback)
|
||||
{
|
||||
struct dynamic_page_pool **pool_list;
|
||||
int i;
|
||||
int ret;
|
||||
|
||||
pool_list = kmalloc_array(NUM_ORDERS, sizeof(*pool_list), GFP_KERNEL);
|
||||
if (!pool_list)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
for (i = 0; i < NUM_ORDERS; i++) {
|
||||
pool_list[i] = dynamic_page_pool_create(order_flags[i],
|
||||
orders[i]);
|
||||
pool_list[i]->vmid = vmid;
|
||||
pool_list[i]->prerelease_callback = callback;
|
||||
atomic_set(&pool_list[i]->count, 0);
|
||||
pool_list[i]->last_low_watermark_ktime = 0;
|
||||
|
||||
if (IS_ERR_OR_NULL(pool_list[i])) {
|
||||
int j;
|
||||
|
||||
pr_err("%s: page pool creation failed for the order %u pool!\n",
|
||||
__func__, orders[i]);
|
||||
for (j = 0; j < i; j++)
|
||||
dynamic_page_pool_destroy(pool_list[j]);
|
||||
|
||||
ret = -ENOMEM;
|
||||
goto free_pool_arr;
|
||||
}
|
||||
}
|
||||
|
||||
return pool_list;
|
||||
|
||||
free_pool_arr:
|
||||
kfree(pool_list);
|
||||
|
||||
return ERR_PTR(ret);
|
||||
}
|
||||
|
||||
void dynamic_page_pool_release_pools(struct dynamic_page_pool **pool_list)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < NUM_ORDERS; i++)
|
||||
dynamic_page_pool_destroy(pool_list[i]);
|
||||
|
||||
kfree(pool_list);
|
||||
}
|
||||
|
||||
static struct shrinker pool_shrinker = {
|
||||
.count_objects = dynamic_page_pool_shrink_count,
|
||||
.scan_objects = dynamic_page_pool_shrink_scan,
|
||||
.seeks = DEFAULT_SEEKS,
|
||||
.batch = 0,
|
||||
};
|
||||
|
||||
int dynamic_page_pool_init_shrinker(void)
|
||||
{
|
||||
int ret;
|
||||
static bool registered;
|
||||
|
||||
if (registered)
|
||||
return 0;
|
||||
|
||||
ret = register_shrinker(&pool_shrinker);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
registered = true;
|
||||
return 0;
|
||||
}
|
||||
110
drivers/dma-buf/heaps/qcom_dynamic_page_pool.h
Normal file
110
drivers/dma-buf/heaps/qcom_dynamic_page_pool.h
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
/*
|
||||
* ION Memory Allocator kernel interface header
|
||||
*
|
||||
* Taken from:
|
||||
* include/linux/dynamic_page_pool.h
|
||||
* Git-repo: https://git.linaro.org/people/john.stultz/android-dev.git
|
||||
* Branch: dma-buf-heap-perf
|
||||
* Git-commit: 458ea8030852755867bdc0384aa40f97aba7a572
|
||||
*
|
||||
* Copyright (C) 2011 Google, Inc.
|
||||
*
|
||||
* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifndef _DYN_PAGE_POOL_H
|
||||
#define _DYN_PAGE_POOL_H
|
||||
|
||||
#include <linux/device.h>
|
||||
#include <linux/kref.h>
|
||||
#include <linux/mm_types.h>
|
||||
#include <linux/mutex.h>
|
||||
#include <linux/shrinker.h>
|
||||
#include <linux/types.h>
|
||||
|
||||
#define HIGH_ORDER_GFP (((GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN \
|
||||
| __GFP_NORETRY) & ~__GFP_RECLAIM) \
|
||||
| __GFP_COMP)
|
||||
#define LOW_ORDER_GFP (GFP_HIGHUSER | __GFP_ZERO | __GFP_COMP)
|
||||
|
||||
/*
|
||||
* We add __GFP_NOWARN for order 4 allocations since the core mm/ framework
|
||||
* makes no guarantee of these allocations succeeding.
|
||||
*/
|
||||
static gfp_t order_flags[] = {HIGH_ORDER_GFP, HIGH_ORDER_GFP,
|
||||
LOW_ORDER_GFP};
|
||||
#if defined(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) && !defined(CONFIG_64BIT) && !defined(CONFIG_ARM_LPAE)
|
||||
static const unsigned int orders[] = {8, 4, 0};
|
||||
#else
|
||||
static const unsigned int orders[] = {9, 4, 0};
|
||||
#endif
|
||||
#define NUM_ORDERS ARRAY_SIZE(orders)
|
||||
|
||||
enum dynamic_pool_callback_ret {
|
||||
DYNAMIC_POOL_SUCCESS,
|
||||
DYNAMIC_POOL_FAILURE,
|
||||
};
|
||||
|
||||
struct dynamic_page_pool;
|
||||
|
||||
typedef enum dynamic_pool_callback_ret (*prerelease_callback)(struct dynamic_page_pool *pool,
|
||||
struct list_head *pages,
|
||||
int num_pages);
|
||||
|
||||
/**
|
||||
* struct dynamic_page_pool - pagepool struct
|
||||
* @high_count: number of highmem items in the pool
|
||||
* @low_count: number of lowmem items in the pool
|
||||
* @count: total number of pages/items in the pool
|
||||
* @high_items: list of highmem items
|
||||
* @low_items: list of lowmem items
|
||||
* @last_low_watermark_ktime: most recent time at which the zone watermarks were
|
||||
* low
|
||||
* @refill_worker: kthread used to refill a pool, if applicable
|
||||
* @mutex: lock protecting this struct and especially the count
|
||||
* item list
|
||||
* @gfp_mask: gfp_mask to use from alloc
|
||||
* @order: order of pages in the pool
|
||||
* @list: list node for list of pools
|
||||
* @vmid: the vmid used for this pool
|
||||
* @prerelease_callback: preprocessing function called before pages are
|
||||
* released to buddy
|
||||
*
|
||||
* Allows you to keep a pool of pre allocated pages to use
|
||||
* Keeping a pool of pages that is ready for dma, ie any cached mapping have
|
||||
* been invalidated from the cache, provides a significant performance benefit
|
||||
* on many systems
|
||||
*/
|
||||
struct dynamic_page_pool {
|
||||
int high_count;
|
||||
int low_count;
|
||||
atomic_t count;
|
||||
struct list_head high_items;
|
||||
struct list_head low_items;
|
||||
ktime_t last_low_watermark_ktime;
|
||||
struct task_struct *refill_worker;
|
||||
struct mutex mutex;
|
||||
gfp_t gfp_mask;
|
||||
unsigned int order;
|
||||
struct list_head list;
|
||||
int vmid;
|
||||
prerelease_callback prerelease_callback;
|
||||
};
|
||||
|
||||
struct dynamic_page_pool **dynamic_page_pool_create_pools(int vmid,
|
||||
prerelease_callback callback);
|
||||
void dynamic_page_pool_release_pools(struct dynamic_page_pool **pool_list);
|
||||
|
||||
struct page *dynamic_page_pool_alloc(struct dynamic_page_pool *pool);
|
||||
void dynamic_page_pool_free(struct dynamic_page_pool *pool, struct page *page);
|
||||
|
||||
int dynamic_page_pool_init_shrinker(void);
|
||||
void dynamic_page_pool_shrink_high_and_low(struct dynamic_page_pool **pools_list,
|
||||
int num_pools, int nr_to_scan);
|
||||
int dynamic_page_pool_total(struct dynamic_page_pool *pool, bool high);
|
||||
|
||||
struct page *dynamic_page_pool_remove(struct dynamic_page_pool *pool, bool high);
|
||||
void dynamic_page_pool_add(struct dynamic_page_pool *pool, struct page *page);
|
||||
|
||||
#endif /* _DYN_PAGE_POOL_H */
|
||||
716
drivers/dma-buf/heaps/qcom_secure_system_heap.c
Normal file
716
drivers/dma-buf/heaps/qcom_secure_system_heap.c
Normal file
|
|
@ -0,0 +1,716 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/*
|
||||
* DMABUF System heap exporter
|
||||
* Originally copied from: drivers/dma-buf/heaps/system_heap.c as of commit
|
||||
* 263e38f82cbb ("dma-buf: heaps: Remove redundant heap identifier from system
|
||||
* heap name")
|
||||
*
|
||||
* Additions taken from modifications to drivers/dma-buf/heaps/system-heap.c,
|
||||
* from patches submitted, are listed below:
|
||||
*
|
||||
* Addition that modifies dma_buf ops to use SG tables taken from
|
||||
* drivers/dma-buf/heaps/system-heap.c in:
|
||||
* https://lore.kernel.org/lkml/20201017013255.43568-2-john.stultz@linaro.org/
|
||||
*
|
||||
* Addition that skips unneeded syncs in the dma_buf ops taken from
|
||||
* https://lore.kernel.org/lkml/20201017013255.43568-5-john.stultz@linaro.org/
|
||||
*
|
||||
* Addition that tries to allocate higher order pages taken from
|
||||
* https://lore.kernel.org/lkml/20201017013255.43568-6-john.stultz@linaro.org/
|
||||
*
|
||||
* Addition that implements an uncached heap taken from
|
||||
* https://lore.kernel.org/lkml/20201017013255.43568-8-john.stultz@linaro.org/,
|
||||
* with our own modificaitons made to account for core kernel changes that are
|
||||
* a part of the patch series.
|
||||
*
|
||||
* Pooling functionality taken from:
|
||||
* Git-repo: https://git.linaro.org/people/john.stultz/android-dev.git
|
||||
* Branch: dma-buf-heap-perf
|
||||
* Git-commit: 6f080eb67dce63c6efa57ef564ca4cd762ccebb0
|
||||
* Git-commit: 6fb9593b928c4cb485bef4e88c59c6b9fdf11352
|
||||
*
|
||||
* Branched off from qcom_system_heap.c as of commit a4e135a8e482
|
||||
* ("dt-bindings: ipcc: Add WPSS client to IPCC header") to accommodate secure
|
||||
* pooling.
|
||||
*
|
||||
* Copyright (C) 2011 Google, Inc.
|
||||
* Copyright (C) 2019, 2020 Linaro Ltd.
|
||||
*
|
||||
* Portions based off of Andrew Davis' SRAM heap:
|
||||
* Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
|
||||
* Andrew F. Davis <afd@ti.com>
|
||||
*
|
||||
* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <linux/dma-buf.h>
|
||||
#include <linux/dma-mapping.h>
|
||||
#include <linux/dma-heap.h>
|
||||
#include <linux/err.h>
|
||||
#include <linux/highmem.h>
|
||||
#include <linux/mm.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/scatterlist.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/vmalloc.h>
|
||||
#include <linux/qcom_dma_heap.h>
|
||||
|
||||
#include "qcom_dma_heap_secure_utils.h"
|
||||
#include "qcom_dynamic_page_pool.h"
|
||||
#include "qcom_sg_ops.h"
|
||||
#include "qcom_secure_system_heap.h"
|
||||
#include "qcom_system_heap.h"
|
||||
|
||||
#define MAX_NR_PREFETCH_REGIONS 32
|
||||
|
||||
/*
|
||||
* The video client may not hold the last reference count on the
|
||||
* ion_buffer(s). Delay for a short time after the video client sends
|
||||
* the IOC_DRAIN event to increase the chance that the reference
|
||||
* count drops to zero. Time in milliseconds.
|
||||
*/
|
||||
#define SHRINK_DELAY 1000
|
||||
|
||||
struct prefetch_info {
|
||||
struct list_head list;
|
||||
struct dma_heap *heap;
|
||||
u64 size;
|
||||
bool shrink;
|
||||
};
|
||||
|
||||
static LIST_HEAD(prefetch_list);
|
||||
static DEFINE_SPINLOCK(work_lock);
|
||||
static struct delayed_work prefetch_work;
|
||||
static struct workqueue_struct *prefetch_wq;
|
||||
|
||||
static LIST_HEAD(secure_heaps);
|
||||
|
||||
static inline struct dynamic_page_pool **get_sys_heap_page_pool(void)
|
||||
{
|
||||
static struct dynamic_page_pool **qcom_sys_heap_pools;
|
||||
struct dma_heap *dma_heap;
|
||||
struct qcom_system_heap *sys_heap;
|
||||
|
||||
if (qcom_sys_heap_pools)
|
||||
return qcom_sys_heap_pools;
|
||||
|
||||
dma_heap = dma_heap_find("qcom,system");
|
||||
if (!dma_heap) {
|
||||
pr_err("Unable to find the system heap\n");
|
||||
goto out;
|
||||
}
|
||||
|
||||
sys_heap = (struct qcom_system_heap *) dma_heap_get_drvdata(dma_heap);
|
||||
qcom_sys_heap_pools = sys_heap->pool_list;
|
||||
|
||||
out:
|
||||
return qcom_sys_heap_pools;
|
||||
}
|
||||
|
||||
static struct sg_table *get_pages(u64 size, struct dma_heap *heap)
|
||||
{
|
||||
LIST_HEAD(pages);
|
||||
struct dynamic_page_pool **qcom_sys_heap_pools;
|
||||
unsigned long size_remaining = size;
|
||||
struct page *page, *tmp_page;
|
||||
int max_order = orders[0];
|
||||
int num_pages = 0;
|
||||
struct sg_table *sgt;
|
||||
struct scatterlist *sg;
|
||||
int ret;
|
||||
|
||||
qcom_sys_heap_pools = get_sys_heap_page_pool();
|
||||
if (!qcom_sys_heap_pools) {
|
||||
pr_err("%s: Couldn't obtain the pools for the system heap!\n", __func__);
|
||||
return ERR_PTR(-EINVAL);
|
||||
}
|
||||
|
||||
while (size_remaining > 0) {
|
||||
page = qcom_sys_heap_alloc_largest_available(qcom_sys_heap_pools,
|
||||
size_remaining,
|
||||
max_order);
|
||||
|
||||
if (!page) {
|
||||
pr_err("%s: Failed to get pages from the system heap: %d, %d!\n",
|
||||
__func__, size_remaining, size);
|
||||
ret = -ENOMEM;
|
||||
goto err;
|
||||
}
|
||||
|
||||
list_add_tail(&page->lru, &pages);
|
||||
size_remaining -= page_size(page);
|
||||
max_order = compound_order(page);
|
||||
num_pages++;
|
||||
}
|
||||
|
||||
sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
|
||||
if (!sgt) {
|
||||
ret = -ENOMEM;
|
||||
goto err;
|
||||
}
|
||||
|
||||
ret = sg_alloc_table(sgt, num_pages, GFP_KERNEL);
|
||||
if (ret) {
|
||||
pr_err("%s: sg table allocation failed with %d\n", __func__, ret);
|
||||
goto free_table;
|
||||
}
|
||||
|
||||
sg = sgt->sgl;
|
||||
list_for_each_entry(page, &pages, lru) {
|
||||
sg_set_page(sg, page, page_size(page), 0);
|
||||
sg = sg_next(sg);
|
||||
}
|
||||
|
||||
dma_map_sgtable(dma_heap_get_dev(heap), sgt, DMA_BIDIRECTIONAL, 0);
|
||||
dma_unmap_sgtable(dma_heap_get_dev(heap), sgt, DMA_BIDIRECTIONAL, 0);
|
||||
|
||||
return sgt;
|
||||
|
||||
free_table:
|
||||
kfree(sgt);
|
||||
err:
|
||||
list_for_each_entry_safe(page, tmp_page, &pages, lru)
|
||||
__free_pages(page, compound_order(page));
|
||||
|
||||
return ERR_PTR(ret);
|
||||
}
|
||||
|
||||
static void process_one_prefetch(struct prefetch_info *info)
|
||||
{
|
||||
struct qcom_secure_system_heap *secure_heap = dma_heap_get_drvdata(info->heap);
|
||||
|
||||
struct sg_table *sgt;
|
||||
struct scatterlist *sg;
|
||||
struct page *page;
|
||||
int ret;
|
||||
int i, j;
|
||||
|
||||
sgt = get_pages(info->size, info->heap);
|
||||
if (IS_ERR(sgt))
|
||||
return;
|
||||
|
||||
ret = hyp_assign_sg_from_flags(sgt, secure_heap->vmid, true);
|
||||
if (ret)
|
||||
goto err;
|
||||
|
||||
for_each_sgtable_sg(sgt, sg, i) {
|
||||
page = sg_page(sg);
|
||||
|
||||
for (j = 0; j < NUM_ORDERS; j++) {
|
||||
if (compound_order(page) == orders[j])
|
||||
break;
|
||||
}
|
||||
dynamic_page_pool_free(secure_heap->pool_list[j], page);
|
||||
}
|
||||
|
||||
sg_free_table(sgt);
|
||||
|
||||
return;
|
||||
|
||||
err:
|
||||
for_each_sgtable_sg(sgt, sg, i) {
|
||||
page = sg_page(sg);
|
||||
__free_pages(page, compound_order(page));
|
||||
}
|
||||
sg_free_table(sgt);
|
||||
}
|
||||
|
||||
static void process_one_shrink(struct prefetch_info *info)
|
||||
{
|
||||
struct qcom_secure_system_heap *secure_heap = dma_heap_get_drvdata(info->heap);
|
||||
|
||||
dynamic_page_pool_shrink_high_and_low(secure_heap->pool_list,
|
||||
NUM_ORDERS, info->size / PAGE_SIZE);
|
||||
}
|
||||
|
||||
static void secure_system_heap_prefetch_work(struct work_struct *work)
|
||||
{
|
||||
struct prefetch_info *info, *tmp;
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&work_lock, flags);
|
||||
list_for_each_entry_safe(info, tmp, &prefetch_list, list) {
|
||||
list_del(&info->list);
|
||||
spin_unlock_irqrestore(&work_lock, flags);
|
||||
|
||||
if (info->shrink)
|
||||
process_one_shrink(info);
|
||||
else
|
||||
process_one_prefetch(info);
|
||||
|
||||
kfree(info);
|
||||
spin_lock_irqsave(&work_lock, flags);
|
||||
}
|
||||
spin_unlock_irqrestore(&work_lock, flags);
|
||||
}
|
||||
|
||||
static int alloc_prefetch_info(struct dma_buf_heap_prefetch_region *region,
|
||||
bool shrink, struct list_head *items)
|
||||
{
|
||||
struct prefetch_info *info;
|
||||
|
||||
info = kmalloc(sizeof(*info), GFP_KERNEL);
|
||||
if (!info)
|
||||
return -ENOMEM;
|
||||
|
||||
info->size = PAGE_ALIGN(region->size);
|
||||
info->heap = region->heap;
|
||||
info->shrink = shrink;
|
||||
INIT_LIST_HEAD(&info->list);
|
||||
list_add_tail(&info->list, items);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int __qcom_secure_system_heap_resize(struct dma_buf_heap_prefetch_region *regions,
|
||||
size_t nr_regions,
|
||||
bool shrink)
|
||||
{
|
||||
struct qcom_secure_system_heap *list_ptr, *secure_heap = NULL;
|
||||
int i, ret = 0;
|
||||
struct prefetch_info *info, *tmp;
|
||||
unsigned long flags;
|
||||
LIST_HEAD(items);
|
||||
|
||||
if (!regions || nr_regions > MAX_NR_PREFETCH_REGIONS) {
|
||||
pr_err("%s: Invalid input for %s\n", __func__,
|
||||
(shrink) ? "drain" : "prefetch");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
for (i = 0; i < nr_regions; i++) {
|
||||
list_for_each_entry(list_ptr, &secure_heaps, list) {
|
||||
if (list_ptr == dma_heap_get_drvdata(regions->heap)) {
|
||||
secure_heap = list_ptr;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!secure_heap) {
|
||||
pr_err("%s: %s is not a secure system heap!\n", __func__,
|
||||
dma_heap_get_name(regions->heap));
|
||||
ret = -EINVAL;
|
||||
goto out_free;
|
||||
}
|
||||
|
||||
ret = alloc_prefetch_info(®ions[i], shrink, &items);
|
||||
if (ret)
|
||||
goto out_free;
|
||||
}
|
||||
|
||||
spin_lock_irqsave(&work_lock, flags);
|
||||
list_splice_tail_init(&items, &prefetch_list);
|
||||
queue_delayed_work(prefetch_wq, &prefetch_work,
|
||||
shrink ? msecs_to_jiffies(SHRINK_DELAY) : 0);
|
||||
spin_unlock_irqrestore(&work_lock, flags);
|
||||
|
||||
return 0;
|
||||
|
||||
out_free:
|
||||
list_for_each_entry_safe(info, tmp, &items, list) {
|
||||
list_del(&info->list);
|
||||
kfree(info);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
int qcom_secure_system_heap_prefetch(struct dma_buf_heap_prefetch_region *regions,
|
||||
size_t nr_regions)
|
||||
{
|
||||
return __qcom_secure_system_heap_resize(regions, nr_regions, false);
|
||||
}
|
||||
EXPORT_SYMBOL(qcom_secure_system_heap_prefetch);
|
||||
|
||||
int qcom_secure_system_heap_drain(struct dma_buf_heap_prefetch_region *regions,
|
||||
size_t nr_regions)
|
||||
{
|
||||
return __qcom_secure_system_heap_resize(regions, nr_regions, true);
|
||||
}
|
||||
EXPORT_SYMBOL(qcom_secure_system_heap_drain);
|
||||
|
||||
static enum dynamic_pool_callback_ret free_secure_pages(struct dynamic_page_pool *pool,
|
||||
struct list_head *pages,
|
||||
int num_pages)
|
||||
{
|
||||
struct page *page;
|
||||
struct sg_table sgt;
|
||||
struct scatterlist *sg;
|
||||
int ret;
|
||||
|
||||
if (!num_pages)
|
||||
return DYNAMIC_POOL_SUCCESS;
|
||||
|
||||
ret = sg_alloc_table(&sgt, num_pages, GFP_KERNEL);
|
||||
if (ret)
|
||||
return DYNAMIC_POOL_FAILURE;
|
||||
|
||||
sg = sgt.sgl;
|
||||
list_for_each_entry(page, pages, lru) {
|
||||
sg_set_page(sg, page, page_size(page), 0);
|
||||
sg = sg_next(sg);
|
||||
}
|
||||
|
||||
ret = hyp_unassign_sg_from_flags(&sgt, pool->vmid, true);
|
||||
sg_free_table(&sgt);
|
||||
|
||||
if (ret)
|
||||
return DYNAMIC_POOL_FAILURE;
|
||||
|
||||
return DYNAMIC_POOL_SUCCESS;
|
||||
}
|
||||
|
||||
static void page_list_merge(struct list_head *secure_pages,
|
||||
struct list_head *non_secure_pages,
|
||||
struct scatterlist *sg,
|
||||
struct scatterlist *non_secure_sg)
|
||||
{
|
||||
bool is_secure_head, is_non_secure_head;
|
||||
struct page *secure_page = list_first_entry(secure_pages, struct page, lru);
|
||||
struct page *non_secure_page = list_first_entry(non_secure_pages, struct page, lru);
|
||||
|
||||
do {
|
||||
is_secure_head = list_entry_is_head(secure_page, secure_pages, lru);
|
||||
is_non_secure_head = list_entry_is_head(non_secure_page, non_secure_pages, lru);
|
||||
|
||||
if (!is_secure_head && !is_non_secure_head) {
|
||||
if (page_size(non_secure_page) >= page_size(secure_page)) {
|
||||
sg_set_page(sg, non_secure_page,
|
||||
page_size(non_secure_page), 0);
|
||||
sg_set_page(non_secure_sg, non_secure_page,
|
||||
page_size(non_secure_page), 0);
|
||||
|
||||
non_secure_page = list_next_entry(non_secure_page, lru);
|
||||
non_secure_sg = sg_next(non_secure_sg);
|
||||
} else {
|
||||
sg_set_page(sg, secure_page,
|
||||
page_size(secure_page), 0);
|
||||
secure_page = list_next_entry(secure_page, lru);
|
||||
}
|
||||
} else if (!is_non_secure_head) {
|
||||
sg_set_page(sg, non_secure_page,
|
||||
page_size(non_secure_page), 0);
|
||||
sg_set_page(non_secure_sg, non_secure_page,
|
||||
page_size(non_secure_page), 0);
|
||||
|
||||
non_secure_page = list_next_entry(non_secure_page, lru);
|
||||
non_secure_sg = sg_next(non_secure_sg);
|
||||
} else if (!is_secure_head) {
|
||||
sg_set_page(sg, secure_page,
|
||||
page_size(secure_page), 0);
|
||||
secure_page = list_next_entry(secure_page, lru);
|
||||
}
|
||||
|
||||
sg = sg_next(sg);
|
||||
} while (sg);
|
||||
}
|
||||
|
||||
static int page_to_pool_ind(struct page *page)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < NUM_ORDERS; i++)
|
||||
if (compound_order(page) == orders[i])
|
||||
break;
|
||||
return i;
|
||||
}
|
||||
|
||||
static void system_heap_free(struct qcom_sg_buffer *buffer)
|
||||
{
|
||||
struct qcom_secure_system_heap *sys_heap;
|
||||
struct sg_table *table;
|
||||
struct scatterlist *sg;
|
||||
int i, j;
|
||||
|
||||
sys_heap = dma_heap_get_drvdata(buffer->heap);
|
||||
table = &buffer->sg_table;
|
||||
|
||||
for_each_sg(table->sgl, sg, table->nents, i) {
|
||||
struct page *page = sg_page(sg);
|
||||
|
||||
for (j = 0; j < NUM_ORDERS; j++) {
|
||||
if (compound_order(page) == orders[j])
|
||||
break;
|
||||
}
|
||||
dynamic_page_pool_free(sys_heap->pool_list[j], page);
|
||||
}
|
||||
sg_free_table(table);
|
||||
kfree(buffer);
|
||||
}
|
||||
|
||||
static struct page *alloc_largest_available(struct dynamic_page_pool **pools,
|
||||
unsigned long size,
|
||||
unsigned int max_order,
|
||||
bool *page_from_secure_pool)
|
||||
{
|
||||
struct page *page = NULL;
|
||||
int i;
|
||||
struct dynamic_page_pool **qcom_sys_heap_pools = get_sys_heap_page_pool();
|
||||
|
||||
if (!qcom_sys_heap_pools) {
|
||||
pr_err("%s: Couldn't obtain the pools for the system heap!\n", __func__);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
*page_from_secure_pool = true;
|
||||
|
||||
for (i = 0; i < NUM_ORDERS; i++) {
|
||||
if (size < (PAGE_SIZE << orders[i]))
|
||||
continue;
|
||||
if (max_order < orders[i])
|
||||
continue;
|
||||
|
||||
mutex_lock(&pools[i]->mutex);
|
||||
if (pools[i]->high_count)
|
||||
page = dynamic_page_pool_remove(pools[i], true);
|
||||
else if (pools[i]->low_count)
|
||||
page = dynamic_page_pool_remove(pools[i], false);
|
||||
mutex_unlock(&pools[i]->mutex);
|
||||
|
||||
if (!page)
|
||||
continue;
|
||||
return page;
|
||||
}
|
||||
|
||||
*page_from_secure_pool = false;
|
||||
|
||||
return qcom_sys_heap_alloc_largest_available(qcom_sys_heap_pools,
|
||||
size,
|
||||
max_order);
|
||||
}
|
||||
|
||||
static struct dma_buf *system_heap_allocate(struct dma_heap *heap,
|
||||
unsigned long len,
|
||||
unsigned long fd_flags,
|
||||
unsigned long heap_flags)
|
||||
{
|
||||
struct qcom_secure_system_heap *sys_heap;
|
||||
struct qcom_sg_buffer *buffer;
|
||||
DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
|
||||
unsigned long size_remaining = len;
|
||||
unsigned int max_order = orders[0];
|
||||
bool page_from_secure_pool;
|
||||
struct dma_buf *dmabuf;
|
||||
struct sg_table *table, non_secure_table;
|
||||
struct scatterlist *sg, *non_secure_sg = NULL;
|
||||
LIST_HEAD(secure_pages);
|
||||
LIST_HEAD(non_secure_pages);
|
||||
struct page *page, *tmp_page;
|
||||
int total_pages = 0, num_non_secure_pages = 0;
|
||||
int ret = -ENOMEM, hyp_ret = 0;
|
||||
int perms;
|
||||
int vmid;
|
||||
|
||||
buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
|
||||
if (!buffer)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
sys_heap = dma_heap_get_drvdata(heap);
|
||||
|
||||
INIT_LIST_HEAD(&buffer->attachments);
|
||||
mutex_init(&buffer->lock);
|
||||
buffer->heap = heap;
|
||||
buffer->len = len;
|
||||
buffer->uncached = true;
|
||||
buffer->free = system_heap_free;
|
||||
|
||||
while (size_remaining > 0) {
|
||||
/*
|
||||
* Avoid trying to allocate memory if the process
|
||||
* has been killed by SIGKILL
|
||||
*/
|
||||
if (fatal_signal_pending(current))
|
||||
goto free_buffer;
|
||||
|
||||
page = alloc_largest_available(sys_heap->pool_list,
|
||||
size_remaining,
|
||||
max_order,
|
||||
&page_from_secure_pool);
|
||||
if (!page)
|
||||
goto free_buffer;
|
||||
|
||||
if (page_from_secure_pool) {
|
||||
list_add_tail(&page->lru, &secure_pages);
|
||||
} else {
|
||||
num_non_secure_pages++;
|
||||
list_add_tail(&page->lru, &non_secure_pages);
|
||||
}
|
||||
|
||||
size_remaining -= page_size(page);
|
||||
max_order = compound_order(page);
|
||||
total_pages++;
|
||||
}
|
||||
|
||||
table = &buffer->sg_table;
|
||||
if (sg_alloc_table(table, total_pages, GFP_KERNEL))
|
||||
goto free_buffer;
|
||||
sg = table->sgl;
|
||||
|
||||
if (num_non_secure_pages) {
|
||||
if (sg_alloc_table(&non_secure_table, num_non_secure_pages, GFP_KERNEL))
|
||||
goto free_sg;
|
||||
non_secure_sg = non_secure_table.sgl;
|
||||
}
|
||||
|
||||
page_list_merge(&secure_pages, &non_secure_pages, sg, non_secure_sg);
|
||||
|
||||
if (num_non_secure_pages) {
|
||||
dma_map_sgtable(dma_heap_get_dev(heap), &non_secure_table, DMA_BIDIRECTIONAL, 0);
|
||||
dma_unmap_sgtable(dma_heap_get_dev(heap), &non_secure_table, DMA_BIDIRECTIONAL, 0);
|
||||
|
||||
hyp_ret = hyp_assign_sg_from_flags(&non_secure_table, sys_heap->vmid, true);
|
||||
if (hyp_ret)
|
||||
goto free_non_secure_sg;
|
||||
}
|
||||
|
||||
perms = msm_secure_get_vmid_perms(sys_heap->vmid);
|
||||
vmid = get_secure_vmid(sys_heap->vmid);
|
||||
buffer->vmperm = mem_buf_vmperm_alloc_staticvm(table,
|
||||
&vmid, &perms, 1);
|
||||
|
||||
if (IS_ERR(buffer->vmperm)) {
|
||||
ret = PTR_ERR(buffer->vmperm);
|
||||
goto hyp_unassign;
|
||||
}
|
||||
|
||||
/* create the dmabuf */
|
||||
exp_info.exp_name = dma_heap_get_name(heap);
|
||||
exp_info.ops = &qcom_sg_buf_ops.dma_ops;
|
||||
exp_info.size = buffer->len;
|
||||
exp_info.flags = fd_flags;
|
||||
exp_info.priv = buffer;
|
||||
dmabuf = mem_buf_dma_buf_export(&exp_info, &qcom_sg_buf_ops);
|
||||
if (IS_ERR(dmabuf)) {
|
||||
ret = PTR_ERR(dmabuf);
|
||||
goto vmperm_release;
|
||||
}
|
||||
|
||||
if (num_non_secure_pages)
|
||||
sg_free_table(&non_secure_table);
|
||||
|
||||
return dmabuf;
|
||||
|
||||
vmperm_release:
|
||||
mem_buf_vmperm_release(buffer->vmperm);
|
||||
|
||||
hyp_unassign:
|
||||
/* We check PagePrivate() below to see if we've reclaimed a particular page */
|
||||
hyp_ret = hyp_unassign_sg_from_flags(&non_secure_table, sys_heap->vmid, true);
|
||||
|
||||
free_non_secure_sg:
|
||||
if (num_non_secure_pages)
|
||||
sg_free_table(&non_secure_table);
|
||||
free_sg:
|
||||
sg_free_table(table);
|
||||
free_buffer:
|
||||
list_for_each_entry_safe(page, tmp_page, &secure_pages, lru)
|
||||
dynamic_page_pool_free(sys_heap->pool_list[page_to_pool_ind(page)], page);
|
||||
if (!hyp_ret)
|
||||
list_for_each_entry_safe(page, tmp_page, &non_secure_pages, lru)
|
||||
__free_pages(page, compound_order(page));
|
||||
|
||||
kfree(buffer);
|
||||
|
||||
return ERR_PTR(ret);
|
||||
}
|
||||
|
||||
|
||||
static const struct dma_heap_ops system_heap_ops = {
|
||||
.allocate = system_heap_allocate,
|
||||
};
|
||||
|
||||
static int create_prefetch_workqueue(void)
|
||||
{
|
||||
static bool registered;
|
||||
|
||||
if (registered)
|
||||
return 0;
|
||||
|
||||
INIT_DELAYED_WORK(&prefetch_work,
|
||||
secure_system_heap_prefetch_work);
|
||||
|
||||
prefetch_wq = alloc_workqueue("system_secure_prefetch_wq",
|
||||
WQ_UNBOUND | WQ_FREEZABLE, 0);
|
||||
if (!prefetch_wq) {
|
||||
pr_err("Failed to create system secure prefetch workqueue\n");
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
registered = true;
|
||||
return 0;
|
||||
}
|
||||
|
||||
void qcom_secure_system_heap_create(const char *name, const char *secure_system_alias, int vmid)
|
||||
{
|
||||
struct dma_heap_export_info exp_info;
|
||||
struct dma_heap *heap;
|
||||
struct qcom_secure_system_heap *sys_heap;
|
||||
int ret;
|
||||
|
||||
if (get_secure_vmid(vmid) == -EINVAL || hweight_long(vmid) != 1) {
|
||||
pr_err("Invalid VMID or supplied more than one VMID\n");
|
||||
ret = -EINVAL;
|
||||
goto out;
|
||||
}
|
||||
|
||||
ret = dynamic_page_pool_init_shrinker();
|
||||
if (ret)
|
||||
goto out;
|
||||
|
||||
ret = create_prefetch_workqueue();
|
||||
if (ret)
|
||||
goto out;
|
||||
|
||||
sys_heap = kzalloc(sizeof(*sys_heap), GFP_KERNEL);
|
||||
if (!sys_heap) {
|
||||
ret = -ENOMEM;
|
||||
goto out;
|
||||
}
|
||||
|
||||
sys_heap->vmid = vmid;
|
||||
sys_heap->pool_list = dynamic_page_pool_create_pools(vmid, free_secure_pages);
|
||||
if (IS_ERR(sys_heap->pool_list)) {
|
||||
ret = PTR_ERR(sys_heap->pool_list);
|
||||
goto free_heap;
|
||||
}
|
||||
|
||||
exp_info.name = name;
|
||||
exp_info.ops = &system_heap_ops;
|
||||
exp_info.priv = sys_heap;
|
||||
|
||||
heap = dma_heap_add(&exp_info);
|
||||
if (IS_ERR(heap)) {
|
||||
ret = PTR_ERR(heap);
|
||||
goto free_pools;
|
||||
}
|
||||
|
||||
dma_coerce_mask_and_coherent(dma_heap_get_dev(heap), DMA_BIT_MASK(64));
|
||||
|
||||
list_add(&sys_heap->list, &secure_heaps);
|
||||
|
||||
pr_info("%s: DMA-BUF Heap: Created '%s'\n", __func__, name);
|
||||
|
||||
if (secure_system_alias != NULL) {
|
||||
exp_info.name = secure_system_alias;
|
||||
|
||||
heap = dma_heap_add(&exp_info);
|
||||
if (IS_ERR(heap)) {
|
||||
pr_err("%s: Failed to create '%s', error is %d\n", __func__,
|
||||
secure_system_alias, PTR_ERR(heap));
|
||||
return;
|
||||
}
|
||||
|
||||
dma_coerce_mask_and_coherent(dma_heap_get_dev(heap), DMA_BIT_MASK(64));
|
||||
|
||||
pr_info("%s: DMA-BUF Heap: Created '%s'\n", __func__, secure_system_alias);
|
||||
}
|
||||
|
||||
return;
|
||||
|
||||
free_pools:
|
||||
dynamic_page_pool_release_pools(sys_heap->pool_list);
|
||||
|
||||
free_heap:
|
||||
kfree(sys_heap);
|
||||
|
||||
out:
|
||||
pr_err("%s: Failed to create '%s', error is %d\n", __func__, name, ret);
|
||||
}
|
||||
30
drivers/dma-buf/heaps/qcom_secure_system_heap.h
Normal file
30
drivers/dma-buf/heaps/qcom_secure_system_heap.h
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifndef _QCOM_SECURE_SYSTEM_HEAP_H
|
||||
#define _QCOM_SECURE_SYSTEM_HEAP_H
|
||||
|
||||
#include <linux/dma-heap.h>
|
||||
#include <linux/err.h>
|
||||
#include "qcom_dynamic_page_pool.h"
|
||||
|
||||
struct qcom_secure_system_heap {
|
||||
struct dynamic_page_pool **pool_list;
|
||||
int vmid;
|
||||
struct list_head list;
|
||||
};
|
||||
|
||||
#ifdef CONFIG_QCOM_DMABUF_HEAPS_SYSTEM_SECURE
|
||||
void qcom_secure_system_heap_create(const char *name, const char *secure_system_alias,
|
||||
int vmid);
|
||||
#else
|
||||
static void qcom_secure_system_heap_create(const char *name, const char *secure_system_alias,
|
||||
int vmid)
|
||||
{
|
||||
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* _QCOM_SECURE_SYSTEM_HEAP_H */
|
||||
544
drivers/dma-buf/heaps/qcom_sg_ops.c
Normal file
544
drivers/dma-buf/heaps/qcom_sg_ops.c
Normal file
|
|
@ -0,0 +1,544 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* mmap() algorithm taken from drivers/staging/android/ion/ion_heap.c as
|
||||
* of commit a3ec289e74b4 ("arm-smmu: Fix missing qsmmuv500 callback")
|
||||
*
|
||||
* Copyright (C) 2011 Google, Inc.
|
||||
* Copyright (C) 2019, 2020 Linaro Ltd.
|
||||
*
|
||||
* Portions based off of Andrew Davis' SRAM heap:
|
||||
* Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
|
||||
* Andrew F. Davis <afd@ti.com>
|
||||
*
|
||||
* These ops were base of the ops in drivers/dma-buf/heaps/system-heap.c from
|
||||
* https://lore.kernel.org/lkml/20201017013255.43568-2-john.stultz@linaro.org/
|
||||
*
|
||||
* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <linux/dma-buf.h>
|
||||
#include <linux/dma-mapping.h>
|
||||
#include <linux/dma-heap.h>
|
||||
#include <linux/err.h>
|
||||
#include <linux/highmem.h>
|
||||
#include <linux/mm.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/scatterlist.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/vmalloc.h>
|
||||
#include <linux/qcom_dma_heap.h>
|
||||
#include <linux/msm_dma_iommu_mapping.h>
|
||||
|
||||
#include "qcom_sg_ops.h"
|
||||
|
||||
static struct sg_table *dup_sg_table(struct sg_table *table)
|
||||
{
|
||||
struct sg_table *new_table;
|
||||
int ret, i;
|
||||
struct scatterlist *sg, *new_sg;
|
||||
|
||||
new_table = kzalloc(sizeof(*new_table), GFP_KERNEL);
|
||||
if (!new_table)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
ret = sg_alloc_table(new_table, table->orig_nents, GFP_KERNEL);
|
||||
if (ret) {
|
||||
kfree(new_table);
|
||||
return ERR_PTR(-ENOMEM);
|
||||
}
|
||||
|
||||
new_sg = new_table->sgl;
|
||||
for_each_sgtable_sg(table, sg, i) {
|
||||
sg_set_page(new_sg, sg_page(sg), sg->length, sg->offset);
|
||||
new_sg = sg_next(new_sg);
|
||||
}
|
||||
|
||||
return new_table;
|
||||
}
|
||||
|
||||
static int qcom_sg_attach(struct dma_buf *dmabuf,
|
||||
struct dma_buf_attachment *attachment)
|
||||
{
|
||||
struct qcom_sg_buffer *buffer = dmabuf->priv;
|
||||
struct dma_heap_attachment *a;
|
||||
struct sg_table *table;
|
||||
|
||||
a = kzalloc(sizeof(*a), GFP_KERNEL);
|
||||
if (!a)
|
||||
return -ENOMEM;
|
||||
|
||||
table = dup_sg_table(&buffer->sg_table);
|
||||
if (IS_ERR(table)) {
|
||||
kfree(a);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
a->table = table;
|
||||
a->dev = attachment->dev;
|
||||
INIT_LIST_HEAD(&a->list);
|
||||
a->mapped = false;
|
||||
|
||||
attachment->priv = a;
|
||||
|
||||
mutex_lock(&buffer->lock);
|
||||
list_add(&a->list, &buffer->attachments);
|
||||
mutex_unlock(&buffer->lock);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void qcom_sg_detach(struct dma_buf *dmabuf,
|
||||
struct dma_buf_attachment *attachment)
|
||||
{
|
||||
struct qcom_sg_buffer *buffer = dmabuf->priv;
|
||||
struct dma_heap_attachment *a = attachment->priv;
|
||||
|
||||
mutex_lock(&buffer->lock);
|
||||
list_del(&a->list);
|
||||
mutex_unlock(&buffer->lock);
|
||||
|
||||
sg_free_table(a->table);
|
||||
kfree(a->table);
|
||||
kfree(a);
|
||||
}
|
||||
|
||||
static struct sg_table *qcom_sg_map_dma_buf(struct dma_buf_attachment *attachment,
|
||||
enum dma_data_direction direction)
|
||||
{
|
||||
struct dma_heap_attachment *a = attachment->priv;
|
||||
struct sg_table *table = a->table;
|
||||
struct qcom_sg_buffer *buffer;
|
||||
struct mem_buf_vmperm *vmperm;
|
||||
unsigned long attrs = attachment->dma_map_attrs;
|
||||
int ret;
|
||||
|
||||
buffer = attachment->dmabuf->priv;
|
||||
vmperm = buffer->vmperm;
|
||||
|
||||
/* Prevent map/unmap during begin/end_cpu_access */
|
||||
mutex_lock(&buffer->lock);
|
||||
|
||||
/* Ensure VM permissions are constant while the buffer is mapped */
|
||||
mem_buf_vmperm_pin(vmperm);
|
||||
if (buffer->uncached || !mem_buf_vmperm_can_cmo(vmperm))
|
||||
attrs |= DMA_ATTR_SKIP_CPU_SYNC;
|
||||
|
||||
if (attrs & DMA_ATTR_DELAYED_UNMAP)
|
||||
ret = msm_dma_map_sgtable(attachment->dev, table, direction,
|
||||
attachment->dmabuf, attrs);
|
||||
else
|
||||
ret = dma_map_sgtable(attachment->dev, table, direction, attrs);
|
||||
|
||||
if (ret) {
|
||||
table = ERR_PTR(-ENOMEM);
|
||||
goto err_map_sgtable;
|
||||
}
|
||||
|
||||
a->mapped = true;
|
||||
mutex_unlock(&buffer->lock);
|
||||
return table;
|
||||
|
||||
err_map_sgtable:
|
||||
mem_buf_vmperm_unpin(vmperm);
|
||||
mutex_unlock(&buffer->lock);
|
||||
return table;
|
||||
}
|
||||
|
||||
static void qcom_sg_unmap_dma_buf(struct dma_buf_attachment *attachment,
|
||||
struct sg_table *table,
|
||||
enum dma_data_direction direction)
|
||||
{
|
||||
struct dma_heap_attachment *a = attachment->priv;
|
||||
struct qcom_sg_buffer *buffer;
|
||||
struct mem_buf_vmperm *vmperm;
|
||||
unsigned long attrs = attachment->dma_map_attrs;
|
||||
|
||||
buffer = attachment->dmabuf->priv;
|
||||
vmperm = buffer->vmperm;
|
||||
|
||||
/* Prevent map/unmap during begin/end_cpu_access */
|
||||
mutex_lock(&buffer->lock);
|
||||
|
||||
if (buffer->uncached || !mem_buf_vmperm_can_cmo(vmperm))
|
||||
attrs |= DMA_ATTR_SKIP_CPU_SYNC;
|
||||
|
||||
a->mapped = false;
|
||||
if (attrs & DMA_ATTR_DELAYED_UNMAP)
|
||||
msm_dma_unmap_sgtable(attachment->dev, table, direction,
|
||||
attachment->dmabuf, attrs);
|
||||
else
|
||||
dma_unmap_sgtable(attachment->dev, table, direction, attrs);
|
||||
mem_buf_vmperm_unpin(vmperm);
|
||||
mutex_unlock(&buffer->lock);
|
||||
}
|
||||
|
||||
static int qcom_sg_dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
|
||||
enum dma_data_direction direction)
|
||||
{
|
||||
struct qcom_sg_buffer *buffer = dmabuf->priv;
|
||||
struct dma_heap_attachment *a;
|
||||
|
||||
if (buffer->uncached)
|
||||
return 0;
|
||||
|
||||
mutex_lock(&buffer->lock);
|
||||
|
||||
/* Keep the same behavior as ion by returning 0 instead of -EPERM */
|
||||
if (!mem_buf_vmperm_can_cmo(buffer->vmperm)) {
|
||||
mutex_unlock(&buffer->lock);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (buffer->vmap_cnt)
|
||||
invalidate_kernel_vmap_range(buffer->vaddr, buffer->len);
|
||||
|
||||
list_for_each_entry(a, &buffer->attachments, list) {
|
||||
if (!a->mapped)
|
||||
continue;
|
||||
dma_sync_sgtable_for_cpu(a->dev, a->table, direction);
|
||||
}
|
||||
|
||||
mutex_unlock(&buffer->lock);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int qcom_sg_dma_buf_end_cpu_access(struct dma_buf *dmabuf,
|
||||
enum dma_data_direction direction)
|
||||
{
|
||||
struct qcom_sg_buffer *buffer = dmabuf->priv;
|
||||
struct dma_heap_attachment *a;
|
||||
|
||||
if (buffer->uncached)
|
||||
return 0;
|
||||
|
||||
mutex_lock(&buffer->lock);
|
||||
|
||||
/* Keep the same behavior as ion by returning 0 instead of -EPERM */
|
||||
if (!mem_buf_vmperm_can_cmo(buffer->vmperm)) {
|
||||
mutex_unlock(&buffer->lock);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (buffer->vmap_cnt)
|
||||
flush_kernel_vmap_range(buffer->vaddr, buffer->len);
|
||||
|
||||
list_for_each_entry(a, &buffer->attachments, list) {
|
||||
if (!a->mapped)
|
||||
continue;
|
||||
dma_sync_sgtable_for_device(a->dev, a->table, direction);
|
||||
}
|
||||
mutex_unlock(&buffer->lock);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int sgl_sync_range(struct device *dev, struct scatterlist *sgl,
|
||||
unsigned int nents, unsigned long offset,
|
||||
unsigned long length,
|
||||
enum dma_data_direction dir, bool for_cpu)
|
||||
{
|
||||
int i;
|
||||
struct scatterlist *sg;
|
||||
unsigned int len = 0;
|
||||
dma_addr_t sg_dma_addr;
|
||||
|
||||
for_each_sg(sgl, sg, nents, i) {
|
||||
if (sg_dma_len(sg) == 0)
|
||||
break;
|
||||
|
||||
if (i > 0) {
|
||||
pr_warn_ratelimited("Partial cmo only supported with 1 segment\n"
|
||||
"is dma_set_max_seg_size being set on dev:%s\n",
|
||||
dev_name(dev));
|
||||
return -EINVAL;
|
||||
}
|
||||
}
|
||||
|
||||
for_each_sg(sgl, sg, nents, i) {
|
||||
unsigned int sg_offset, sg_left, size = 0;
|
||||
|
||||
if (i == 0)
|
||||
sg_dma_addr = sg_dma_address(sg);
|
||||
|
||||
len += sg->length;
|
||||
if (len <= offset) {
|
||||
sg_dma_addr += sg->length;
|
||||
continue;
|
||||
}
|
||||
|
||||
sg_left = len - offset;
|
||||
sg_offset = sg->length - sg_left;
|
||||
|
||||
size = (length < sg_left) ? length : sg_left;
|
||||
if (for_cpu)
|
||||
dma_sync_single_range_for_cpu(dev, sg_dma_addr,
|
||||
sg_offset, size, dir);
|
||||
else
|
||||
dma_sync_single_range_for_device(dev, sg_dma_addr,
|
||||
sg_offset, size, dir);
|
||||
|
||||
offset += size;
|
||||
length -= size;
|
||||
sg_dma_addr += sg->length;
|
||||
|
||||
if (length == 0)
|
||||
break;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int qcom_sg_dma_buf_begin_cpu_access_partial(struct dma_buf *dmabuf,
|
||||
enum dma_data_direction dir,
|
||||
unsigned int offset,
|
||||
unsigned int len)
|
||||
{
|
||||
struct qcom_sg_buffer *buffer = dmabuf->priv;
|
||||
struct dma_heap_attachment *a;
|
||||
int ret = 0;
|
||||
|
||||
if (buffer->uncached)
|
||||
return 0;
|
||||
|
||||
mutex_lock(&buffer->lock);
|
||||
|
||||
/* Keep the same behavior as ion by returning 0 instead of -EPERM */
|
||||
if (!mem_buf_vmperm_can_cmo(buffer->vmperm)) {
|
||||
mutex_unlock(&buffer->lock);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (buffer->vmap_cnt)
|
||||
invalidate_kernel_vmap_range(buffer->vaddr + offset, len);
|
||||
|
||||
list_for_each_entry(a, &buffer->attachments, list) {
|
||||
if (!a->mapped)
|
||||
continue;
|
||||
|
||||
ret = sgl_sync_range(a->dev, a->table->sgl, a->table->orig_nents,
|
||||
offset, len, dir, true);
|
||||
}
|
||||
mutex_unlock(&buffer->lock);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int qcom_sg_dma_buf_end_cpu_access_partial(struct dma_buf *dmabuf,
|
||||
enum dma_data_direction direction,
|
||||
unsigned int offset,
|
||||
unsigned int len)
|
||||
{
|
||||
struct qcom_sg_buffer *buffer = dmabuf->priv;
|
||||
struct dma_heap_attachment *a;
|
||||
int ret = 0;
|
||||
|
||||
if (buffer->uncached)
|
||||
return 0;
|
||||
|
||||
mutex_lock(&buffer->lock);
|
||||
|
||||
/* Keep the same behavior as ion by returning 0 instead of -EPERM */
|
||||
if (!mem_buf_vmperm_can_cmo(buffer->vmperm)) {
|
||||
mutex_unlock(&buffer->lock);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (buffer->vmap_cnt)
|
||||
flush_kernel_vmap_range(buffer->vaddr + offset, len);
|
||||
|
||||
list_for_each_entry(a, &buffer->attachments, list) {
|
||||
if (!a->mapped)
|
||||
continue;
|
||||
|
||||
ret = sgl_sync_range(a->dev, a->table->sgl, a->table->orig_nents,
|
||||
offset, len, direction, false);
|
||||
}
|
||||
mutex_unlock(&buffer->lock);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void qcom_sg_vm_ops_open(struct vm_area_struct *vma)
|
||||
{
|
||||
struct mem_buf_vmperm *vmperm = vma->vm_private_data;
|
||||
|
||||
mem_buf_vmperm_pin(vmperm);
|
||||
}
|
||||
|
||||
static void qcom_sg_vm_ops_close(struct vm_area_struct *vma)
|
||||
{
|
||||
struct mem_buf_vmperm *vmperm = vma->vm_private_data;
|
||||
|
||||
mem_buf_vmperm_unpin(vmperm);
|
||||
}
|
||||
|
||||
static const struct vm_operations_struct qcom_sg_vm_ops = {
|
||||
.open = qcom_sg_vm_ops_open,
|
||||
.close = qcom_sg_vm_ops_close,
|
||||
};
|
||||
|
||||
static int qcom_sg_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
|
||||
{
|
||||
struct qcom_sg_buffer *buffer = dmabuf->priv;
|
||||
struct sg_table *table = &buffer->sg_table;
|
||||
struct scatterlist *sg;
|
||||
unsigned long addr = vma->vm_start;
|
||||
unsigned long offset = vma->vm_pgoff * PAGE_SIZE;
|
||||
int ret;
|
||||
int i;
|
||||
|
||||
mem_buf_vmperm_pin(buffer->vmperm);
|
||||
if (!mem_buf_vmperm_can_mmap(buffer->vmperm, vma)) {
|
||||
mem_buf_vmperm_unpin(buffer->vmperm);
|
||||
return -EPERM;
|
||||
}
|
||||
|
||||
vma->vm_ops = &qcom_sg_vm_ops;
|
||||
vma->vm_private_data = buffer->vmperm;
|
||||
if (buffer->uncached)
|
||||
vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
|
||||
|
||||
for_each_sg(table->sgl, sg, table->nents, i) {
|
||||
struct page *page = sg_page(sg);
|
||||
unsigned long remainder = vma->vm_end - addr;
|
||||
unsigned long len = sg->length;
|
||||
|
||||
if (offset >= sg->length) {
|
||||
offset -= sg->length;
|
||||
continue;
|
||||
} else if (offset) {
|
||||
page += offset / PAGE_SIZE;
|
||||
len = sg->length - offset;
|
||||
offset = 0;
|
||||
}
|
||||
len = min(len, remainder);
|
||||
ret = remap_pfn_range(vma, addr, page_to_pfn(page), len,
|
||||
vma->vm_page_prot);
|
||||
if (ret) {
|
||||
mem_buf_vmperm_unpin(buffer->vmperm);
|
||||
return ret;
|
||||
}
|
||||
addr += len;
|
||||
if (addr >= vma->vm_end)
|
||||
return 0;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void *qcom_sg_do_vmap(struct qcom_sg_buffer *buffer)
|
||||
{
|
||||
struct sg_table *table = &buffer->sg_table;
|
||||
int npages = PAGE_ALIGN(buffer->len) / PAGE_SIZE;
|
||||
struct page **pages = vmalloc(sizeof(struct page *) * npages);
|
||||
struct page **tmp = pages;
|
||||
struct sg_page_iter piter;
|
||||
pgprot_t pgprot = PAGE_KERNEL;
|
||||
void *vaddr;
|
||||
|
||||
if (!pages)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
if (buffer->uncached)
|
||||
pgprot = pgprot_writecombine(PAGE_KERNEL);
|
||||
|
||||
for_each_sgtable_page(table, &piter, 0) {
|
||||
WARN_ON(tmp - pages >= npages);
|
||||
*tmp++ = sg_page_iter_page(&piter);
|
||||
}
|
||||
|
||||
vaddr = vmap(pages, npages, VM_MAP, pgprot);
|
||||
vfree(pages);
|
||||
|
||||
if (!vaddr)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
return vaddr;
|
||||
}
|
||||
|
||||
static int qcom_sg_vmap(struct dma_buf *dmabuf, struct dma_buf_map *map)
|
||||
{
|
||||
struct qcom_sg_buffer *buffer = dmabuf->priv;
|
||||
void *vaddr;
|
||||
int ret = 0;
|
||||
|
||||
mem_buf_vmperm_pin(buffer->vmperm);
|
||||
if (!mem_buf_vmperm_can_vmap(buffer->vmperm)) {
|
||||
mem_buf_vmperm_unpin(buffer->vmperm);
|
||||
return -EPERM;
|
||||
}
|
||||
|
||||
mutex_lock(&buffer->lock);
|
||||
if (buffer->vmap_cnt) {
|
||||
buffer->vmap_cnt++;
|
||||
dma_buf_map_set_vaddr(map, buffer->vaddr);
|
||||
goto out;
|
||||
}
|
||||
|
||||
vaddr = qcom_sg_do_vmap(buffer);
|
||||
if (IS_ERR(vaddr)) {
|
||||
ret = PTR_ERR(vaddr);
|
||||
mem_buf_vmperm_unpin(buffer->vmperm);
|
||||
goto out;
|
||||
}
|
||||
|
||||
buffer->vaddr = vaddr;
|
||||
buffer->vmap_cnt++;
|
||||
dma_buf_map_set_vaddr(map, buffer->vaddr);
|
||||
out:
|
||||
mutex_unlock(&buffer->lock);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void qcom_sg_vunmap(struct dma_buf *dmabuf, struct dma_buf_map *map)
|
||||
{
|
||||
struct qcom_sg_buffer *buffer = dmabuf->priv;
|
||||
|
||||
mutex_lock(&buffer->lock);
|
||||
if (!--buffer->vmap_cnt) {
|
||||
vunmap(buffer->vaddr);
|
||||
buffer->vaddr = NULL;
|
||||
}
|
||||
mem_buf_vmperm_unpin(buffer->vmperm);
|
||||
mutex_unlock(&buffer->lock);
|
||||
dma_buf_map_clear(map);
|
||||
}
|
||||
|
||||
static void qcom_sg_release(struct dma_buf *dmabuf)
|
||||
{
|
||||
struct qcom_sg_buffer *buffer = dmabuf->priv;
|
||||
|
||||
if (mem_buf_vmperm_release(buffer->vmperm))
|
||||
return;
|
||||
|
||||
msm_dma_buf_freed(buffer);
|
||||
buffer->free(buffer);
|
||||
}
|
||||
|
||||
static struct mem_buf_vmperm *qcom_sg_lookup_vmperm(struct dma_buf *dmabuf)
|
||||
{
|
||||
struct qcom_sg_buffer *buffer = dmabuf->priv;
|
||||
|
||||
return buffer->vmperm;
|
||||
}
|
||||
|
||||
struct mem_buf_dma_buf_ops qcom_sg_buf_ops = {
|
||||
.attach = qcom_sg_attach,
|
||||
.lookup = qcom_sg_lookup_vmperm,
|
||||
.dma_ops = {
|
||||
.attach = NULL, /* Will be set by mem_buf_dma_buf_export */
|
||||
.detach = qcom_sg_detach,
|
||||
.map_dma_buf = qcom_sg_map_dma_buf,
|
||||
.unmap_dma_buf = qcom_sg_unmap_dma_buf,
|
||||
.begin_cpu_access = qcom_sg_dma_buf_begin_cpu_access,
|
||||
.end_cpu_access = qcom_sg_dma_buf_end_cpu_access,
|
||||
.begin_cpu_access_partial = qcom_sg_dma_buf_begin_cpu_access_partial,
|
||||
.end_cpu_access_partial = qcom_sg_dma_buf_end_cpu_access_partial,
|
||||
.mmap = qcom_sg_mmap,
|
||||
.vmap = qcom_sg_vmap,
|
||||
.vunmap = qcom_sg_vunmap,
|
||||
.release = qcom_sg_release,
|
||||
}
|
||||
};
|
||||
EXPORT_SYMBOL(qcom_sg_buf_ops);
|
||||
43
drivers/dma-buf/heaps/qcom_sg_ops.h
Normal file
43
drivers/dma-buf/heaps/qcom_sg_ops.h
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (C) 2011 Google, Inc.
|
||||
* Copyright (C) 2019, 2020 Linaro Ltd.
|
||||
*
|
||||
* Portions based off of Andrew Davis' SRAM heap:
|
||||
* Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
|
||||
* Andrew F. Davis <afd@ti.com>
|
||||
*
|
||||
* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifndef _QCOM_SG_OPS_H
|
||||
#define _QCOM_SG_OPS_H
|
||||
|
||||
#include <linux/scatterlist.h>
|
||||
#include <linux/dma-heap.h>
|
||||
#include <linux/device.h>
|
||||
#include <linux/mem-buf-exporter.h>
|
||||
|
||||
struct qcom_sg_buffer {
|
||||
struct dma_heap *heap;
|
||||
struct list_head attachments;
|
||||
struct mutex lock;
|
||||
unsigned long len;
|
||||
struct sg_table sg_table;
|
||||
int vmap_cnt;
|
||||
void *vaddr;
|
||||
bool uncached;
|
||||
struct mem_buf_vmperm *vmperm;
|
||||
void (*free)(struct qcom_sg_buffer *buffer);
|
||||
};
|
||||
|
||||
struct dma_heap_attachment {
|
||||
struct device *dev;
|
||||
struct sg_table *table;
|
||||
struct list_head list;
|
||||
bool mapped;
|
||||
};
|
||||
|
||||
extern struct mem_buf_dma_buf_ops qcom_sg_buf_ops;
|
||||
|
||||
#endif /* _QCOM_SG_OPS_H */
|
||||
584
drivers/dma-buf/heaps/qcom_system_heap.c
Normal file
584
drivers/dma-buf/heaps/qcom_system_heap.c
Normal file
|
|
@ -0,0 +1,584 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/*
|
||||
* DMABUF System heap exporter
|
||||
* Originally copied from: drivers/dma-buf/heaps/system_heap.c as of commit
|
||||
* 263e38f82cbb ("dma-buf: heaps: Remove redundant heap identifier from system
|
||||
* heap name")
|
||||
*
|
||||
* Additions taken from modifications to drivers/dma-buf/heaps/system-heap.c,
|
||||
* from patches submitted, are listed below:
|
||||
*
|
||||
* Addition that modifies dma_buf ops to use SG tables taken from
|
||||
* drivers/dma-buf/heaps/system-heap.c in:
|
||||
* https://lore.kernel.org/lkml/20201017013255.43568-2-john.stultz@linaro.org/
|
||||
*
|
||||
* Addition that skips unneeded syncs in the dma_buf ops taken from
|
||||
* https://lore.kernel.org/lkml/20201017013255.43568-5-john.stultz@linaro.org/
|
||||
*
|
||||
* Addition that tries to allocate higher order pages taken from
|
||||
* https://lore.kernel.org/lkml/20201017013255.43568-6-john.stultz@linaro.org/
|
||||
*
|
||||
* Addition that implements an uncached heap taken from
|
||||
* https://lore.kernel.org/lkml/20201017013255.43568-8-john.stultz@linaro.org/,
|
||||
* with our own modificaitons made to account for core kernel changes that are
|
||||
* a part of the patch series.
|
||||
*
|
||||
* Pooling functionality taken from:
|
||||
* Git-repo: https://git.linaro.org/people/john.stultz/android-dev.git
|
||||
* Branch: dma-buf-heap-perf
|
||||
* Git-commit: 6f080eb67dce63c6efa57ef564ca4cd762ccebb0
|
||||
* Git-commit: 6fb9593b928c4cb485bef4e88c59c6b9fdf11352
|
||||
*
|
||||
* Copyright (C) 2011 Google, Inc.
|
||||
* Copyright (C) 2019, 2020 Linaro Ltd.
|
||||
*
|
||||
* Portions based off of Andrew Davis' SRAM heap:
|
||||
* Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
|
||||
* Andrew F. Davis <afd@ti.com>
|
||||
*
|
||||
* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <linux/dma-buf.h>
|
||||
#include <linux/dma-mapping.h>
|
||||
#include <linux/dma-heap.h>
|
||||
#include <linux/err.h>
|
||||
#include <linux/highmem.h>
|
||||
#include <linux/mm.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/scatterlist.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/vmalloc.h>
|
||||
#include <linux/kthread.h>
|
||||
#include <linux/qcom_dma_heap.h>
|
||||
#include <uapi/linux/sched/types.h>
|
||||
|
||||
#include "qcom_dma_heap_secure_utils.h"
|
||||
#include "qcom_dynamic_page_pool.h"
|
||||
#include "qcom_sg_ops.h"
|
||||
#include "qcom_system_heap.h"
|
||||
|
||||
#if IS_ENABLED(CONFIG_QCOM_DMABUF_HEAPS_PAGE_POOL_REFILL)
|
||||
#define DYNAMIC_POOL_FILL_MARK (100 * SZ_1M)
|
||||
#define DYNAMIC_POOL_LOW_MARK_PERCENT 40UL
|
||||
#define DYNAMIC_POOL_LOW_MARK ((DYNAMIC_POOL_FILL_MARK * DYNAMIC_POOL_LOW_MARK_PERCENT) / 100)
|
||||
|
||||
#define DYNAMIC_POOL_REFILL_DEFER_WINDOW_MS 10
|
||||
#define DYNAMIC_POOL_KTHREAD_NICE_VAL 10
|
||||
|
||||
static int get_dynamic_pool_fillmark(struct dynamic_page_pool *pool)
|
||||
{
|
||||
return DYNAMIC_POOL_FILL_MARK / (PAGE_SIZE << pool->order);
|
||||
}
|
||||
|
||||
static bool dynamic_pool_fillmark_reached(struct dynamic_page_pool *pool)
|
||||
{
|
||||
return atomic_read(&pool->count) >= get_dynamic_pool_fillmark(pool);
|
||||
}
|
||||
|
||||
static int get_dynamic_pool_lowmark(struct dynamic_page_pool *pool)
|
||||
{
|
||||
return DYNAMIC_POOL_LOW_MARK / (PAGE_SIZE << pool->order);
|
||||
}
|
||||
|
||||
static bool dynamic_pool_count_below_lowmark(struct dynamic_page_pool *pool)
|
||||
{
|
||||
return atomic_read(&pool->count) < get_dynamic_pool_lowmark(pool);
|
||||
}
|
||||
|
||||
/* Based on gfp_zone() in mm/mmzone.c since it is not exported. */
|
||||
static enum zone_type dynamic_pool_gfp_zone(gfp_t flags)
|
||||
{
|
||||
enum zone_type z;
|
||||
gfp_t local_flags = flags;
|
||||
int bit;
|
||||
|
||||
bit = (__force int) ((local_flags) & GFP_ZONEMASK);
|
||||
|
||||
z = (GFP_ZONE_TABLE >> (bit * GFP_ZONES_SHIFT)) &
|
||||
((1 << GFP_ZONES_SHIFT) - 1);
|
||||
VM_BUG_ON((GFP_ZONE_BAD >> bit) & 1);
|
||||
return z;
|
||||
}
|
||||
|
||||
/*
|
||||
* Based on __zone_watermark_ok() in mm/page_alloc.c since it is not exported.
|
||||
*
|
||||
* Return true if free base pages are above 'mark'. For high-order checks it
|
||||
* will return true of the order-0 watermark is reached and there is at least
|
||||
* one free page of a suitable size. Checking now avoids taking the zone lock
|
||||
* to check in the allocation paths if no pages are free.
|
||||
*/
|
||||
static bool __dynamic_pool_zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
|
||||
int highest_zoneidx, long free_pages)
|
||||
{
|
||||
long min = mark;
|
||||
long unusable_free;
|
||||
int o;
|
||||
|
||||
/*
|
||||
* Access to high atomic reserves is not required, and CMA should not be
|
||||
* used, since these allocations are non-movable.
|
||||
*/
|
||||
unusable_free = ((1 << order) - 1) + z->nr_reserved_highatomic;
|
||||
#ifdef CONFIG_CMA
|
||||
unusable_free += zone_page_state(z, NR_FREE_CMA_PAGES);
|
||||
#endif
|
||||
|
||||
/* free_pages may go negative - that's OK */
|
||||
free_pages -= unusable_free;
|
||||
|
||||
/*
|
||||
* Check watermarks for an order-0 allocation request. If these
|
||||
* are not met, then a high-order request also cannot go ahead
|
||||
* even if a suitable page happened to be free.
|
||||
*
|
||||
* 'min' can be taken as 'mark' since we do not expect these allocations
|
||||
* to require disruptive actions (such as running the OOM killer) or
|
||||
* a lot of effort.
|
||||
*/
|
||||
if (free_pages <= min + z->lowmem_reserve[highest_zoneidx])
|
||||
return false;
|
||||
|
||||
/* If this is an order-0 request then the watermark is fine */
|
||||
if (!order)
|
||||
return true;
|
||||
|
||||
/* For a high-order request, check at least one suitable page is free */
|
||||
for (o = order; o < MAX_ORDER; o++) {
|
||||
struct free_area *area = &z->free_area[o];
|
||||
int mt;
|
||||
|
||||
if (!area->nr_free)
|
||||
continue;
|
||||
|
||||
for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) {
|
||||
if (!free_area_empty(area, mt))
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Based on zone_watermark_ok_safe from mm/page_alloc.c since it is not exported. */
|
||||
static bool dynamic_pool_zone_watermark_ok_safe(struct zone *z, unsigned int order,
|
||||
unsigned long mark, int highest_zoneidx)
|
||||
{
|
||||
long free_pages = zone_page_state(z, NR_FREE_PAGES);
|
||||
|
||||
if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
|
||||
free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);
|
||||
|
||||
return __dynamic_pool_zone_watermark_ok(z, order, mark, highest_zoneidx, free_pages);
|
||||
}
|
||||
|
||||
/* do a simple check to see if we are in any low memory situation */
|
||||
static bool dynamic_pool_refill_ok(struct dynamic_page_pool *pool)
|
||||
{
|
||||
struct zone *zone;
|
||||
int i, mark;
|
||||
enum zone_type classzone_idx = dynamic_pool_gfp_zone(pool->gfp_mask);
|
||||
s64 delta;
|
||||
|
||||
/* check if we are within the refill defer window */
|
||||
delta = ktime_ms_delta(ktime_get(), pool->last_low_watermark_ktime);
|
||||
if (delta < DYNAMIC_POOL_REFILL_DEFER_WINDOW_MS)
|
||||
return false;
|
||||
|
||||
/*
|
||||
* make sure that if we allocate a pool->order page from buddy,
|
||||
* we don't put the zone watermarks below the high threshold.
|
||||
* This makes sure there's no unwanted repetitive refilling and
|
||||
* reclaiming of buddy pages on the pool.
|
||||
*/
|
||||
for (i = classzone_idx; i >= 0; i--) {
|
||||
zone = &NODE_DATA(numa_node_id())->node_zones[i];
|
||||
|
||||
if (!strcmp(zone->name, "DMA32"))
|
||||
continue;
|
||||
|
||||
mark = high_wmark_pages(zone);
|
||||
mark += 1 << pool->order;
|
||||
if (!dynamic_pool_zone_watermark_ok_safe(zone, pool->order, mark, classzone_idx)) {
|
||||
pool->last_low_watermark_ktime = ktime_get();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static void dynamic_page_pool_refill(struct dynamic_page_pool *pool)
|
||||
{
|
||||
struct page *page;
|
||||
gfp_t gfp_refill = (pool->gfp_mask | __GFP_RECLAIM) & ~__GFP_NORETRY;
|
||||
|
||||
/* skip refilling order 0 pools */
|
||||
if (!pool->order)
|
||||
return;
|
||||
|
||||
while (!dynamic_pool_fillmark_reached(pool) && dynamic_pool_refill_ok(pool)) {
|
||||
page = alloc_pages(gfp_refill, pool->order);
|
||||
if (!page)
|
||||
break;
|
||||
|
||||
dynamic_page_pool_add(pool, page);
|
||||
}
|
||||
}
|
||||
|
||||
static bool dynamic_pool_needs_refill(struct dynamic_page_pool *pool)
|
||||
{
|
||||
return pool->order && dynamic_pool_count_below_lowmark(pool);
|
||||
}
|
||||
|
||||
static int system_heap_refill_worker(void *data)
|
||||
{
|
||||
struct dynamic_page_pool **pool_list = data;
|
||||
int i;
|
||||
|
||||
for (;;) {
|
||||
for (i = 0; i < NUM_ORDERS; i++) {
|
||||
if (dynamic_pool_count_below_lowmark(pool_list[i]))
|
||||
dynamic_page_pool_refill(pool_list[i]);
|
||||
}
|
||||
|
||||
set_current_state(TASK_INTERRUPTIBLE);
|
||||
if (unlikely(kthread_should_stop())) {
|
||||
set_current_state(TASK_RUNNING);
|
||||
break;
|
||||
}
|
||||
schedule();
|
||||
|
||||
set_current_state(TASK_RUNNING);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int system_heap_create_refill_worker(struct qcom_system_heap *sys_heap, const char *name)
|
||||
{
|
||||
struct task_struct *refill_worker;
|
||||
struct sched_attr attr = { .sched_nice = DYNAMIC_POOL_KTHREAD_NICE_VAL };
|
||||
int ret;
|
||||
int i;
|
||||
|
||||
refill_worker = kthread_run(system_heap_refill_worker, sys_heap->pool_list,
|
||||
"%s-pool-refill-thread", name);
|
||||
if (IS_ERR(refill_worker)) {
|
||||
pr_err("%s: failed to create %s-pool-refill-thread: %ld\n",
|
||||
__func__, name, PTR_ERR(refill_worker));
|
||||
return PTR_ERR(refill_worker);
|
||||
}
|
||||
|
||||
ret = sched_setattr(refill_worker, &attr);
|
||||
if (ret) {
|
||||
pr_warn("%s: failed to set task priority for %s-pool-refill-thread: ret = %d\n",
|
||||
__func__, name, ret);
|
||||
kthread_stop(refill_worker);
|
||||
return ret;
|
||||
}
|
||||
|
||||
for (i = 0; i < NUM_ORDERS; i++)
|
||||
sys_heap->pool_list[i]->refill_worker = refill_worker;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void system_heap_destroy_refill_worker(struct qcom_system_heap *sys_heap)
|
||||
{
|
||||
kthread_stop(sys_heap->pool_list[0]->refill_worker);
|
||||
}
|
||||
#else
|
||||
static bool dynamic_pool_needs_refill(struct dynamic_page_pool *pool)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static int system_heap_create_refill_worker(struct qcom_system_heap *sys_heap, const char *name)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void system_heap_destroy_refill_worker(struct qcom_system_heap *sys_heap)
|
||||
{
|
||||
}
|
||||
#endif
|
||||
|
||||
static int system_heap_clear_pages(struct page **pages, int num, pgprot_t pgprot)
|
||||
{
|
||||
void *addr = vmap(pages, num, VM_MAP, pgprot);
|
||||
|
||||
if (!addr)
|
||||
return -ENOMEM;
|
||||
memset(addr, 0, PAGE_SIZE * num);
|
||||
vunmap(addr);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int system_heap_zero_buffer(struct qcom_sg_buffer *buffer)
|
||||
{
|
||||
struct sg_table *sgt = &buffer->sg_table;
|
||||
struct sg_page_iter piter;
|
||||
struct page *pages[32];
|
||||
int p = 0;
|
||||
int ret = 0;
|
||||
|
||||
for_each_sgtable_page(sgt, &piter, 0) {
|
||||
pages[p++] = sg_page_iter_page(&piter);
|
||||
if (p == ARRAY_SIZE(pages)) {
|
||||
ret = system_heap_clear_pages(pages, p, PAGE_KERNEL);
|
||||
if (ret)
|
||||
return ret;
|
||||
p = 0;
|
||||
}
|
||||
}
|
||||
if (p)
|
||||
ret = system_heap_clear_pages(pages, p, PAGE_KERNEL);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void system_heap_free(struct qcom_sg_buffer *buffer)
|
||||
{
|
||||
struct qcom_system_heap *sys_heap;
|
||||
struct sg_table *table;
|
||||
struct scatterlist *sg;
|
||||
int i, j;
|
||||
|
||||
sys_heap = dma_heap_get_drvdata(buffer->heap);
|
||||
table = &buffer->sg_table;
|
||||
|
||||
/* Zero the buffer pages before adding back to the pool */
|
||||
system_heap_zero_buffer(buffer);
|
||||
|
||||
for_each_sg(table->sgl, sg, table->nents, i) {
|
||||
struct page *page = sg_page(sg);
|
||||
|
||||
for (j = 0; j < NUM_ORDERS; j++) {
|
||||
if (compound_order(page) == orders[j])
|
||||
break;
|
||||
}
|
||||
dynamic_page_pool_free(sys_heap->pool_list[j], page);
|
||||
}
|
||||
sg_free_table(table);
|
||||
kfree(buffer);
|
||||
}
|
||||
|
||||
struct page *qcom_sys_heap_alloc_largest_available(struct dynamic_page_pool **pools,
|
||||
unsigned long size,
|
||||
unsigned int max_order)
|
||||
{
|
||||
struct page *page = NULL;
|
||||
int i;
|
||||
|
||||
for (i = 0; i < NUM_ORDERS; i++) {
|
||||
if (size < (PAGE_SIZE << orders[i]))
|
||||
continue;
|
||||
if (max_order < orders[i])
|
||||
continue;
|
||||
|
||||
mutex_lock(&pools[i]->mutex);
|
||||
if (pools[i]->high_count)
|
||||
page = dynamic_page_pool_remove(pools[i], true);
|
||||
else if (pools[i]->low_count)
|
||||
page = dynamic_page_pool_remove(pools[i], false);
|
||||
mutex_unlock(&pools[i]->mutex);
|
||||
|
||||
if (!page)
|
||||
page = alloc_pages(pools[i]->gfp_mask, pools[i]->order);
|
||||
if (!page)
|
||||
continue;
|
||||
|
||||
if (dynamic_pool_needs_refill(pools[i]))
|
||||
wake_up_process(pools[i]->refill_worker);
|
||||
|
||||
return page;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static struct dma_buf *system_heap_allocate(struct dma_heap *heap,
|
||||
unsigned long len,
|
||||
unsigned long fd_flags,
|
||||
unsigned long heap_flags)
|
||||
{
|
||||
struct qcom_system_heap *sys_heap;
|
||||
struct qcom_sg_buffer *buffer;
|
||||
DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
|
||||
unsigned long size_remaining = len;
|
||||
unsigned int max_order = orders[0];
|
||||
struct dma_buf *dmabuf;
|
||||
struct sg_table *table;
|
||||
struct scatterlist *sg;
|
||||
struct list_head pages;
|
||||
struct page *page, *tmp_page;
|
||||
int i, ret = -ENOMEM;
|
||||
|
||||
buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
|
||||
if (!buffer)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
sys_heap = dma_heap_get_drvdata(heap);
|
||||
|
||||
INIT_LIST_HEAD(&buffer->attachments);
|
||||
mutex_init(&buffer->lock);
|
||||
buffer->heap = heap;
|
||||
buffer->len = len;
|
||||
buffer->uncached = sys_heap->uncached;
|
||||
buffer->free = system_heap_free;
|
||||
|
||||
INIT_LIST_HEAD(&pages);
|
||||
i = 0;
|
||||
while (size_remaining > 0) {
|
||||
/*
|
||||
* Avoid trying to allocate memory if the process
|
||||
* has been killed by SIGKILL
|
||||
*/
|
||||
if (fatal_signal_pending(current))
|
||||
goto free_buffer;
|
||||
|
||||
page = qcom_sys_heap_alloc_largest_available(sys_heap->pool_list,
|
||||
size_remaining,
|
||||
max_order);
|
||||
if (!page)
|
||||
goto free_buffer;
|
||||
|
||||
list_add_tail(&page->lru, &pages);
|
||||
size_remaining -= page_size(page);
|
||||
max_order = compound_order(page);
|
||||
i++;
|
||||
}
|
||||
|
||||
table = &buffer->sg_table;
|
||||
if (sg_alloc_table(table, i, GFP_KERNEL))
|
||||
goto free_buffer;
|
||||
|
||||
sg = table->sgl;
|
||||
list_for_each_entry_safe(page, tmp_page, &pages, lru) {
|
||||
sg_set_page(sg, page, page_size(page), 0);
|
||||
sg = sg_next(sg);
|
||||
list_del(&page->lru);
|
||||
}
|
||||
|
||||
/*
|
||||
* For uncached buffers, we need to initially flush cpu cache, since
|
||||
* the __GFP_ZERO on the allocation means the zeroing was done by the
|
||||
* cpu and thus it is likely cached. Map (and implicitly flush) and
|
||||
* unmap it now so we don't get corruption later on.
|
||||
*/
|
||||
if (buffer->uncached) {
|
||||
dma_map_sgtable(dma_heap_get_dev(heap), table, DMA_BIDIRECTIONAL, 0);
|
||||
dma_unmap_sgtable(dma_heap_get_dev(heap), table, DMA_BIDIRECTIONAL, 0);
|
||||
}
|
||||
|
||||
buffer->vmperm = mem_buf_vmperm_alloc(table);
|
||||
|
||||
if (IS_ERR(buffer->vmperm)) {
|
||||
ret = PTR_ERR(buffer->vmperm);
|
||||
goto free_sg;
|
||||
}
|
||||
|
||||
/* create the dmabuf */
|
||||
exp_info.exp_name = dma_heap_get_name(heap);
|
||||
exp_info.size = buffer->len;
|
||||
exp_info.flags = fd_flags;
|
||||
exp_info.priv = buffer;
|
||||
dmabuf = mem_buf_dma_buf_export(&exp_info, &qcom_sg_buf_ops);
|
||||
if (IS_ERR(dmabuf)) {
|
||||
ret = PTR_ERR(dmabuf);
|
||||
goto vmperm_release;
|
||||
}
|
||||
|
||||
return dmabuf;
|
||||
|
||||
vmperm_release:
|
||||
mem_buf_vmperm_release(buffer->vmperm);
|
||||
free_sg:
|
||||
sg_free_table(table);
|
||||
free_buffer:
|
||||
list_for_each_entry_safe(page, tmp_page, &pages, lru)
|
||||
__free_pages(page, compound_order(page));
|
||||
kfree(buffer);
|
||||
|
||||
return ERR_PTR(ret);
|
||||
}
|
||||
|
||||
static const struct dma_heap_ops system_heap_ops = {
|
||||
.allocate = system_heap_allocate,
|
||||
};
|
||||
|
||||
void qcom_system_heap_create(const char *name, const char *system_alias, bool uncached)
|
||||
{
|
||||
struct dma_heap_export_info exp_info;
|
||||
struct dma_heap *heap;
|
||||
struct qcom_system_heap *sys_heap;
|
||||
int ret;
|
||||
|
||||
ret = dynamic_page_pool_init_shrinker();
|
||||
if (ret)
|
||||
goto out;
|
||||
|
||||
sys_heap = kzalloc(sizeof(*sys_heap), GFP_KERNEL);
|
||||
if (!sys_heap) {
|
||||
ret = -ENOMEM;
|
||||
goto out;
|
||||
}
|
||||
|
||||
exp_info.name = name;
|
||||
exp_info.ops = &system_heap_ops;
|
||||
exp_info.priv = sys_heap;
|
||||
|
||||
sys_heap->uncached = uncached;
|
||||
|
||||
sys_heap->pool_list = dynamic_page_pool_create_pools(0, NULL);
|
||||
if (IS_ERR(sys_heap->pool_list)) {
|
||||
ret = PTR_ERR(sys_heap->pool_list);
|
||||
goto free_heap;
|
||||
}
|
||||
|
||||
ret = system_heap_create_refill_worker(sys_heap, name);
|
||||
if (ret)
|
||||
goto free_pools;
|
||||
|
||||
heap = dma_heap_add(&exp_info);
|
||||
if (IS_ERR(heap)) {
|
||||
ret = PTR_ERR(heap);
|
||||
goto stop_worker;
|
||||
}
|
||||
|
||||
if (uncached)
|
||||
dma_coerce_mask_and_coherent(dma_heap_get_dev(heap),
|
||||
DMA_BIT_MASK(64));
|
||||
|
||||
pr_info("%s: DMA-BUF Heap: Created '%s'\n", __func__, name);
|
||||
|
||||
if (system_alias != NULL) {
|
||||
exp_info.name = system_alias;
|
||||
|
||||
heap = dma_heap_add(&exp_info);
|
||||
if (IS_ERR(heap)) {
|
||||
pr_err("%s: Failed to create '%s', error is %d\n", __func__,
|
||||
system_alias, PTR_ERR(heap));
|
||||
return;
|
||||
}
|
||||
|
||||
dma_coerce_mask_and_coherent(dma_heap_get_dev(heap), DMA_BIT_MASK(64));
|
||||
|
||||
pr_info("%s: DMA-BUF Heap: Created '%s'\n", __func__, system_alias);
|
||||
}
|
||||
|
||||
return;
|
||||
|
||||
stop_worker:
|
||||
system_heap_destroy_refill_worker(sys_heap);
|
||||
|
||||
free_pools:
|
||||
dynamic_page_pool_release_pools(sys_heap->pool_list);
|
||||
|
||||
free_heap:
|
||||
kfree(sys_heap);
|
||||
|
||||
out:
|
||||
pr_err("%s: Failed to create '%s', error is %d\n", __func__, name, ret);
|
||||
}
|
||||
37
drivers/dma-buf/heaps/qcom_system_heap.h
Normal file
37
drivers/dma-buf/heaps/qcom_system_heap.h
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifndef _QCOM_SYSTEM_HEAP_H
|
||||
#define _QCOM_SYSTEM_HEAP_H
|
||||
|
||||
#include <linux/dma-heap.h>
|
||||
#include <linux/err.h>
|
||||
#include "qcom_dynamic_page_pool.h"
|
||||
|
||||
struct qcom_system_heap {
|
||||
int uncached;
|
||||
struct dynamic_page_pool **pool_list;
|
||||
};
|
||||
|
||||
#ifdef CONFIG_QCOM_DMABUF_HEAPS_SYSTEM
|
||||
void qcom_system_heap_create(const char *name, const char *system_alias, bool uncached);
|
||||
struct page *qcom_sys_heap_alloc_largest_available(struct dynamic_page_pool **pools,
|
||||
unsigned long size,
|
||||
unsigned int max_order);
|
||||
#else
|
||||
struct page *qcom_sys_heap_alloc_largest_available(struct dynamic_page_pool **pools,
|
||||
unsigned long size,
|
||||
unsigned int max_order)
|
||||
{
|
||||
return ERR_PTR(-EOPNOTSUPP);
|
||||
}
|
||||
|
||||
static void qcom_system_heap_create(const char *name, const char *system_alias, bool uncached)
|
||||
{
|
||||
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* _QCOM_SYSTEM_HEAP_H */
|
||||
480
drivers/iommu/msm_dma_iommu_mapping.c
Normal file
480
drivers/iommu/msm_dma_iommu_mapping.c
Normal file
|
|
@ -0,0 +1,480 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Copyright (c) 2015-2019, 2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <linux/dma-map-ops.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/kref.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/rbtree.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/mutex.h>
|
||||
#include <linux/err.h>
|
||||
#include <asm/barrier.h>
|
||||
|
||||
#include <linux/msm_dma_iommu_mapping.h>
|
||||
#include <linux/qcom-dma-mapping.h>
|
||||
|
||||
/**
|
||||
* struct msm_iommu_map - represents a mapping of an ion buffer to an iommu
|
||||
* @lnode - list node to exist in the buffer's list of iommu mappings
|
||||
* @dev - Device this is mapped to. Used as key
|
||||
* @sgl - The scatterlist for this mapping
|
||||
* @nents - Number of entries in sgl
|
||||
* @dir - The direction for the map.
|
||||
* @meta - Backpointer to the meta this guy belongs to.
|
||||
* @ref - for reference counting this mapping
|
||||
* @attrs - dma mapping attributes
|
||||
* @buf_start_addr - address of start of buffer
|
||||
*
|
||||
* Represents a mapping of one dma_buf buffer to a particular device
|
||||
* and address range. There may exist other mappings of this buffer in
|
||||
* different devices. All mappings will have the same cacheability and security.
|
||||
*/
|
||||
struct msm_iommu_map {
|
||||
struct list_head lnode;
|
||||
struct rb_node node;
|
||||
struct device *dev;
|
||||
struct scatterlist *sgl;
|
||||
unsigned int nents;
|
||||
enum dma_data_direction dir;
|
||||
struct msm_iommu_meta *meta;
|
||||
struct kref ref;
|
||||
unsigned long attrs;
|
||||
dma_addr_t buf_start_addr;
|
||||
};
|
||||
|
||||
struct msm_iommu_meta {
|
||||
struct rb_node node;
|
||||
struct list_head iommu_maps;
|
||||
struct kref ref;
|
||||
struct mutex lock;
|
||||
void *buffer;
|
||||
};
|
||||
|
||||
static struct rb_root iommu_root;
|
||||
static DEFINE_MUTEX(msm_iommu_map_mutex);
|
||||
|
||||
static void msm_iommu_meta_add(struct msm_iommu_meta *meta)
|
||||
{
|
||||
struct rb_root *root = &iommu_root;
|
||||
struct rb_node **p = &root->rb_node;
|
||||
struct rb_node *parent = NULL;
|
||||
struct msm_iommu_meta *entry;
|
||||
|
||||
while (*p) {
|
||||
parent = *p;
|
||||
entry = rb_entry(parent, struct msm_iommu_meta, node);
|
||||
|
||||
if (meta->buffer < entry->buffer)
|
||||
p = &(*p)->rb_left;
|
||||
else if (meta->buffer > entry->buffer)
|
||||
p = &(*p)->rb_right;
|
||||
else
|
||||
pr_err("%s: dma_buf %pK already exists\n", __func__,
|
||||
entry->buffer);
|
||||
}
|
||||
|
||||
rb_link_node(&meta->node, parent, p);
|
||||
rb_insert_color(&meta->node, root);
|
||||
}
|
||||
|
||||
static struct msm_iommu_meta *msm_iommu_meta_lookup(void *buffer)
|
||||
{
|
||||
struct rb_root *root = &iommu_root;
|
||||
struct rb_node **p = &root->rb_node;
|
||||
struct rb_node *parent = NULL;
|
||||
struct msm_iommu_meta *entry = NULL;
|
||||
|
||||
while (*p) {
|
||||
parent = *p;
|
||||
entry = rb_entry(parent, struct msm_iommu_meta, node);
|
||||
|
||||
if (buffer < entry->buffer)
|
||||
p = &(*p)->rb_left;
|
||||
else if (buffer > entry->buffer)
|
||||
p = &(*p)->rb_right;
|
||||
else
|
||||
return entry;
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void msm_iommu_add(struct msm_iommu_meta *meta,
|
||||
struct msm_iommu_map *iommu)
|
||||
{
|
||||
INIT_LIST_HEAD(&iommu->lnode);
|
||||
list_add(&iommu->lnode, &meta->iommu_maps);
|
||||
}
|
||||
|
||||
|
||||
static struct msm_iommu_map *msm_iommu_lookup(struct msm_iommu_meta *meta,
|
||||
struct device *dev)
|
||||
{
|
||||
struct msm_iommu_map *entry;
|
||||
|
||||
list_for_each_entry(entry, &meta->iommu_maps, lnode) {
|
||||
if (entry->dev == dev)
|
||||
return entry;
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static struct msm_iommu_meta *msm_iommu_meta_create(struct dma_buf *dma_buf)
|
||||
{
|
||||
struct msm_iommu_meta *meta;
|
||||
|
||||
meta = kzalloc(sizeof(*meta), GFP_KERNEL);
|
||||
|
||||
if (!meta)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
INIT_LIST_HEAD(&meta->iommu_maps);
|
||||
meta->buffer = dma_buf->priv;
|
||||
kref_init(&meta->ref);
|
||||
mutex_init(&meta->lock);
|
||||
msm_iommu_meta_add(meta);
|
||||
|
||||
return meta;
|
||||
}
|
||||
|
||||
static void msm_iommu_meta_put(struct msm_iommu_meta *meta);
|
||||
|
||||
static struct scatterlist *clone_sgl(struct scatterlist *sg, int nents)
|
||||
{
|
||||
struct scatterlist *next, *s;
|
||||
int i;
|
||||
struct sg_table table;
|
||||
|
||||
if (sg_alloc_table(&table, nents, GFP_KERNEL))
|
||||
return NULL;
|
||||
next = table.sgl;
|
||||
for_each_sg(sg, s, nents, i) {
|
||||
*next = *s;
|
||||
next = sg_next(next);
|
||||
}
|
||||
return table.sgl;
|
||||
}
|
||||
|
||||
static inline int __msm_dma_map_sg(struct device *dev, struct scatterlist *sg,
|
||||
int nents, enum dma_data_direction dir,
|
||||
struct dma_buf *dma_buf,
|
||||
unsigned long attrs)
|
||||
{
|
||||
struct msm_iommu_map *iommu_map;
|
||||
struct msm_iommu_meta *iommu_meta = NULL;
|
||||
int ret = 0;
|
||||
bool extra_meta_ref_taken = false;
|
||||
int late_unmap = !(attrs & DMA_ATTR_NO_DELAYED_UNMAP);
|
||||
|
||||
mutex_lock(&msm_iommu_map_mutex);
|
||||
iommu_meta = msm_iommu_meta_lookup(dma_buf->priv);
|
||||
|
||||
if (!iommu_meta) {
|
||||
iommu_meta = msm_iommu_meta_create(dma_buf);
|
||||
|
||||
if (IS_ERR(iommu_meta)) {
|
||||
mutex_unlock(&msm_iommu_map_mutex);
|
||||
ret = PTR_ERR(iommu_meta);
|
||||
goto out;
|
||||
}
|
||||
if (late_unmap) {
|
||||
kref_get(&iommu_meta->ref);
|
||||
extra_meta_ref_taken = true;
|
||||
}
|
||||
} else {
|
||||
kref_get(&iommu_meta->ref);
|
||||
}
|
||||
|
||||
mutex_unlock(&msm_iommu_map_mutex);
|
||||
|
||||
mutex_lock(&iommu_meta->lock);
|
||||
iommu_map = msm_iommu_lookup(iommu_meta, dev);
|
||||
if (!iommu_map) {
|
||||
iommu_map = kmalloc(sizeof(*iommu_map), GFP_KERNEL);
|
||||
|
||||
if (!iommu_map) {
|
||||
ret = -ENOMEM;
|
||||
goto out_unlock;
|
||||
}
|
||||
|
||||
ret = dma_map_sg_attrs(dev, sg, nents, dir, attrs);
|
||||
if (!ret) {
|
||||
kfree(iommu_map);
|
||||
goto out_unlock;
|
||||
}
|
||||
|
||||
iommu_map->sgl = clone_sgl(sg, nents);
|
||||
if (!iommu_map->sgl) {
|
||||
kfree(iommu_map);
|
||||
ret = -ENOMEM;
|
||||
goto out_unlock;
|
||||
}
|
||||
iommu_map->nents = nents;
|
||||
iommu_map->dev = dev;
|
||||
iommu_map->dir = dir;
|
||||
iommu_map->attrs = attrs;
|
||||
iommu_map->buf_start_addr = sg_phys(sg);
|
||||
|
||||
kref_init(&iommu_map->ref);
|
||||
if (late_unmap)
|
||||
kref_get(&iommu_map->ref);
|
||||
iommu_map->meta = iommu_meta;
|
||||
msm_iommu_add(iommu_meta, iommu_map);
|
||||
|
||||
} else {
|
||||
if (nents == iommu_map->nents &&
|
||||
dir == iommu_map->dir &&
|
||||
(attrs & ~DMA_ATTR_SKIP_CPU_SYNC) ==
|
||||
(iommu_map->attrs & ~DMA_ATTR_SKIP_CPU_SYNC) &&
|
||||
sg_phys(sg) == iommu_map->buf_start_addr) {
|
||||
struct scatterlist *sg_tmp = sg;
|
||||
struct scatterlist *map_sg;
|
||||
int i;
|
||||
|
||||
for_each_sg(iommu_map->sgl, map_sg, nents, i) {
|
||||
sg_dma_address(sg_tmp) = sg_dma_address(map_sg);
|
||||
sg_dma_len(sg_tmp) = sg_dma_len(map_sg);
|
||||
if (sg_dma_len(map_sg) == 0)
|
||||
break;
|
||||
|
||||
sg_tmp = sg_next(sg_tmp);
|
||||
if (sg_tmp == NULL)
|
||||
break;
|
||||
}
|
||||
|
||||
kref_get(&iommu_map->ref);
|
||||
|
||||
if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
|
||||
dma_sync_sg_for_device(dev, iommu_map->sgl,
|
||||
iommu_map->nents, iommu_map->dir);
|
||||
|
||||
if (dev_is_dma_coherent(dev))
|
||||
/*
|
||||
* Ensure all outstanding changes for coherent
|
||||
* buffers are applied to the cache before any
|
||||
* DMA occurs.
|
||||
*/
|
||||
dmb(ish);
|
||||
ret = nents;
|
||||
} else {
|
||||
bool start_diff = (sg_phys(sg) !=
|
||||
iommu_map->buf_start_addr);
|
||||
|
||||
dev_err(dev, "lazy map request differs:\n"
|
||||
"req dir:%d, original dir:%d\n"
|
||||
"req nents:%d, original nents:%d\n"
|
||||
"req map attrs:%lu, original map attrs:%lu\n"
|
||||
"req buffer start address differs:%d\n",
|
||||
dir, iommu_map->dir, nents,
|
||||
iommu_map->nents, attrs, iommu_map->attrs,
|
||||
start_diff);
|
||||
ret = -EINVAL;
|
||||
}
|
||||
}
|
||||
mutex_unlock(&iommu_meta->lock);
|
||||
return ret;
|
||||
|
||||
out_unlock:
|
||||
mutex_unlock(&iommu_meta->lock);
|
||||
out:
|
||||
if (!IS_ERR(iommu_meta)) {
|
||||
if (extra_meta_ref_taken)
|
||||
msm_iommu_meta_put(iommu_meta);
|
||||
msm_iommu_meta_put(iommu_meta);
|
||||
}
|
||||
return ret;
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
* We are not taking a reference to the dma_buf here. It is expected that
|
||||
* clients hold reference to the dma_buf until they are done with mapping and
|
||||
* unmapping.
|
||||
*/
|
||||
int msm_dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents,
|
||||
enum dma_data_direction dir, struct dma_buf *dma_buf,
|
||||
unsigned long attrs)
|
||||
{
|
||||
int ret;
|
||||
|
||||
if (IS_ERR_OR_NULL(dev)) {
|
||||
pr_err("%s: dev pointer is invalid\n", __func__);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (IS_ERR_OR_NULL(sg)) {
|
||||
pr_err("%s: sg table pointer is invalid\n", __func__);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (IS_ERR_OR_NULL(dma_buf)) {
|
||||
pr_err("%s: dma_buf pointer is invalid\n", __func__);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
ret = __msm_dma_map_sg(dev, sg, nents, dir, dma_buf, attrs);
|
||||
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL(msm_dma_map_sg_attrs);
|
||||
|
||||
static void msm_iommu_meta_destroy(struct kref *kref)
|
||||
{
|
||||
struct msm_iommu_meta *meta = container_of(kref, struct msm_iommu_meta,
|
||||
ref);
|
||||
|
||||
if (!list_empty(&meta->iommu_maps)) {
|
||||
WARN(1, "%s: DMA Buffer %pK being destroyed with outstanding iommu mappings!\n",
|
||||
__func__, meta->buffer);
|
||||
}
|
||||
rb_erase(&meta->node, &iommu_root);
|
||||
kfree(meta);
|
||||
}
|
||||
|
||||
static void msm_iommu_meta_put(struct msm_iommu_meta *meta)
|
||||
{
|
||||
/*
|
||||
* Need to lock here to prevent race against map/unmap
|
||||
*/
|
||||
mutex_lock(&msm_iommu_map_mutex);
|
||||
kref_put(&meta->ref, msm_iommu_meta_destroy);
|
||||
mutex_unlock(&msm_iommu_map_mutex);
|
||||
}
|
||||
|
||||
static void msm_iommu_map_release(struct kref *kref)
|
||||
{
|
||||
struct msm_iommu_map *map = container_of(kref, struct msm_iommu_map,
|
||||
ref);
|
||||
struct sg_table table;
|
||||
|
||||
table.nents = table.orig_nents = map->nents;
|
||||
table.sgl = map->sgl;
|
||||
list_del(&map->lnode);
|
||||
|
||||
/* Skip an additional cache maintenance on the dma unmap path */
|
||||
if (!(map->attrs & DMA_ATTR_SKIP_CPU_SYNC))
|
||||
map->attrs |= DMA_ATTR_SKIP_CPU_SYNC;
|
||||
dma_unmap_sg_attrs(map->dev, map->sgl, map->nents, map->dir,
|
||||
map->attrs);
|
||||
sg_free_table(&table);
|
||||
kfree(map);
|
||||
}
|
||||
|
||||
void msm_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl,
|
||||
int nents, enum dma_data_direction dir,
|
||||
struct dma_buf *dma_buf, unsigned long attrs)
|
||||
{
|
||||
struct msm_iommu_map *iommu_map;
|
||||
struct msm_iommu_meta *meta;
|
||||
|
||||
mutex_lock(&msm_iommu_map_mutex);
|
||||
meta = msm_iommu_meta_lookup(dma_buf->priv);
|
||||
if (!meta) {
|
||||
WARN(1, "%s: (%pK) was never mapped\n", __func__, dma_buf);
|
||||
mutex_unlock(&msm_iommu_map_mutex);
|
||||
goto out;
|
||||
|
||||
}
|
||||
mutex_unlock(&msm_iommu_map_mutex);
|
||||
|
||||
mutex_lock(&meta->lock);
|
||||
iommu_map = msm_iommu_lookup(meta, dev);
|
||||
|
||||
if (!iommu_map) {
|
||||
WARN(1, "%s: (%pK) was never mapped for device %p\n", __func__,
|
||||
dma_buf, dev);
|
||||
mutex_unlock(&meta->lock);
|
||||
goto out;
|
||||
}
|
||||
|
||||
if (dir != iommu_map->dir)
|
||||
WARN(1, "%s: (%pK) dir:%d differs from original dir:%d\n",
|
||||
__func__, dma_buf, dir, iommu_map->dir);
|
||||
|
||||
if (attrs && ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0))
|
||||
dma_sync_sg_for_cpu(dev, iommu_map->sgl, iommu_map->nents, dir);
|
||||
|
||||
iommu_map->attrs = attrs;
|
||||
kref_put(&iommu_map->ref, msm_iommu_map_release);
|
||||
mutex_unlock(&meta->lock);
|
||||
|
||||
msm_iommu_meta_put(meta);
|
||||
|
||||
out:
|
||||
return;
|
||||
}
|
||||
EXPORT_SYMBOL(msm_dma_unmap_sg_attrs);
|
||||
|
||||
int msm_dma_unmap_all_for_dev(struct device *dev)
|
||||
{
|
||||
int ret = 0;
|
||||
struct msm_iommu_meta *meta;
|
||||
struct rb_root *root;
|
||||
struct rb_node *meta_node;
|
||||
|
||||
mutex_lock(&msm_iommu_map_mutex);
|
||||
root = &iommu_root;
|
||||
meta_node = rb_first(root);
|
||||
while (meta_node) {
|
||||
struct msm_iommu_map *iommu_map;
|
||||
struct msm_iommu_map *iommu_map_next;
|
||||
|
||||
meta = rb_entry(meta_node, struct msm_iommu_meta, node);
|
||||
mutex_lock(&meta->lock);
|
||||
list_for_each_entry_safe(iommu_map, iommu_map_next,
|
||||
&meta->iommu_maps, lnode)
|
||||
if (iommu_map->dev == dev)
|
||||
if (!kref_put(&iommu_map->ref,
|
||||
msm_iommu_map_release))
|
||||
ret = -EINVAL;
|
||||
|
||||
mutex_unlock(&meta->lock);
|
||||
meta_node = rb_next(meta_node);
|
||||
}
|
||||
mutex_unlock(&msm_iommu_map_mutex);
|
||||
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL(msm_dma_unmap_all_for_dev);
|
||||
|
||||
/*
|
||||
* Only to be called by ION code when a buffer is freed
|
||||
*/
|
||||
void msm_dma_buf_freed(void *buffer)
|
||||
{
|
||||
struct msm_iommu_map *iommu_map;
|
||||
struct msm_iommu_map *iommu_map_next;
|
||||
struct msm_iommu_meta *meta;
|
||||
|
||||
mutex_lock(&msm_iommu_map_mutex);
|
||||
meta = msm_iommu_meta_lookup(buffer);
|
||||
if (!meta) {
|
||||
/* Already unmapped (assuming no late unmapping) */
|
||||
mutex_unlock(&msm_iommu_map_mutex);
|
||||
return;
|
||||
}
|
||||
mutex_unlock(&msm_iommu_map_mutex);
|
||||
|
||||
mutex_lock(&meta->lock);
|
||||
|
||||
list_for_each_entry_safe(iommu_map, iommu_map_next, &meta->iommu_maps,
|
||||
lnode)
|
||||
kref_put(&iommu_map->ref, msm_iommu_map_release);
|
||||
|
||||
if (!list_empty(&meta->iommu_maps)) {
|
||||
WARN(1, "%s: DMA buffer %pK destroyed with outstanding iommu mappings\n",
|
||||
__func__, meta->buffer);
|
||||
}
|
||||
|
||||
INIT_LIST_HEAD(&meta->iommu_maps);
|
||||
mutex_unlock(&meta->lock);
|
||||
|
||||
msm_iommu_meta_put(meta);
|
||||
}
|
||||
EXPORT_SYMBOL(msm_dma_buf_freed);
|
||||
|
||||
MODULE_LICENSE("GPL v2");
|
||||
144
include/linux/msm_dma_iommu_mapping.h
Normal file
144
include/linux/msm_dma_iommu_mapping.h
Normal file
|
|
@ -0,0 +1,144 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (c) 2015-2016, 2018, 2021 The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifndef _LINUX_MSM_DMA_IOMMU_MAPPING_H
|
||||
#define _LINUX_MSM_DMA_IOMMU_MAPPING_H
|
||||
|
||||
#include <linux/device.h>
|
||||
#include <linux/dma-buf.h>
|
||||
#include <linux/scatterlist.h>
|
||||
#include <linux/dma-mapping.h>
|
||||
#include <linux/qcom-dma-mapping.h>
|
||||
|
||||
#if IS_ENABLED(CONFIG_QCOM_LAZY_MAPPING)
|
||||
/*
|
||||
* This function is not taking a reference to the dma_buf here. It is expected
|
||||
* that clients hold reference to the dma_buf until they are done with mapping
|
||||
* and unmapping.
|
||||
*/
|
||||
int msm_dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents,
|
||||
enum dma_data_direction dir, struct dma_buf *dma_buf,
|
||||
unsigned long attrs);
|
||||
|
||||
/*
|
||||
* This function takes an extra reference to the dma_buf.
|
||||
* What this means is that calling msm_dma_unmap_sg will not result in buffer's
|
||||
* iommu mapping being removed, which means that subsequent calls to lazy map
|
||||
* will simply re-use the existing iommu mapping.
|
||||
* The iommu unmapping of the buffer will occur when the ION buffer is
|
||||
* destroyed.
|
||||
* Using lazy mapping can provide a performance benefit because subsequent
|
||||
* mappings are faster.
|
||||
*
|
||||
* The limitation of using this API are that all subsequent iommu mappings
|
||||
* must be the same as the original mapping, ie they must map the same part of
|
||||
* the buffer with the same dma data direction. Also there can't be multiple
|
||||
* mappings of different parts of the buffer.
|
||||
*/
|
||||
static inline int msm_dma_map_sg_lazy(struct device *dev,
|
||||
struct scatterlist *sg, int nents,
|
||||
enum dma_data_direction dir,
|
||||
struct dma_buf *dma_buf)
|
||||
{
|
||||
return msm_dma_map_sg_attrs(dev, sg, nents, dir, dma_buf, 0);
|
||||
}
|
||||
|
||||
static inline int msm_dma_map_sg(struct device *dev, struct scatterlist *sg,
|
||||
int nents, enum dma_data_direction dir,
|
||||
struct dma_buf *dma_buf)
|
||||
{
|
||||
unsigned long attrs;
|
||||
|
||||
attrs = DMA_ATTR_NO_DELAYED_UNMAP;
|
||||
return msm_dma_map_sg_attrs(dev, sg, nents, dir, dma_buf, attrs);
|
||||
}
|
||||
|
||||
void msm_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl,
|
||||
int nents, enum dma_data_direction dir,
|
||||
struct dma_buf *dma_buf, unsigned long attrs);
|
||||
|
||||
static inline int msm_dma_map_sgtable(struct device *dev, struct sg_table *sgt,
|
||||
enum dma_data_direction dir,
|
||||
struct dma_buf *dma_buf, unsigned long attrs)
|
||||
{
|
||||
int nents;
|
||||
|
||||
nents = msm_dma_map_sg_attrs(dev, sgt->sgl, sgt->orig_nents, dir, dma_buf, attrs);
|
||||
if (nents <= 0)
|
||||
return -EINVAL;
|
||||
|
||||
sgt->nents = nents;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void msm_dma_unmap_sgtable(struct device *dev, struct sg_table *sgt,
|
||||
enum dma_data_direction dir,
|
||||
struct dma_buf *dma_buf, unsigned long attrs)
|
||||
{
|
||||
msm_dma_unmap_sg_attrs(dev, sgt->sgl, sgt->nents, dir, dma_buf, attrs);
|
||||
}
|
||||
|
||||
int msm_dma_unmap_all_for_dev(struct device *dev);
|
||||
|
||||
/*
|
||||
* Below is private function only to be called by framework (ION) and not by
|
||||
* clients.
|
||||
*/
|
||||
void msm_dma_buf_freed(void *buffer);
|
||||
|
||||
#else /*CONFIG_QCOM_LAZY_MAPPING*/
|
||||
|
||||
static inline int msm_dma_map_sg_attrs(struct device *dev,
|
||||
struct scatterlist *sg, int nents,
|
||||
enum dma_data_direction dir, struct dma_buf *dma_buf,
|
||||
unsigned long attrs)
|
||||
{
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
static inline void
|
||||
msm_dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sgl,
|
||||
int nents, enum dma_data_direction dir,
|
||||
struct dma_buf *dma_buf, unsigned long attrs)
|
||||
{
|
||||
}
|
||||
|
||||
static inline int msm_dma_map_sg_lazy(struct device *dev,
|
||||
struct scatterlist *sg, int nents,
|
||||
enum dma_data_direction dir,
|
||||
struct dma_buf *dma_buf)
|
||||
{
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
static inline int msm_dma_map_sg(struct device *dev, struct scatterlist *sg,
|
||||
int nents, enum dma_data_direction dir,
|
||||
struct dma_buf *dma_buf)
|
||||
{
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
static inline int msm_dma_map_sgtable(struct device *dev, struct sg_table *sgt,
|
||||
enum dma_data_direction dir,
|
||||
struct dma_buf *dma_buf, unsigned long attrs)
|
||||
{
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
static inline void msm_dma_unmap_sgtable(struct device *dev, struct sg_table *sgt,
|
||||
enum dma_data_direction dir,
|
||||
struct dma_buf *dma_buf, unsigned long attrs)
|
||||
{
|
||||
}
|
||||
|
||||
static inline int msm_dma_unmap_all_for_dev(struct device *dev)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void msm_dma_buf_freed(void *buffer) {}
|
||||
#endif /*CONFIG_QCOM_LAZY_MAPPING*/
|
||||
|
||||
#endif
|
||||
108
include/linux/qcom_dma_heap.h
Normal file
108
include/linux/qcom_dma_heap.h
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
#ifndef _QCOM_DMA_HEAP_H
|
||||
#define _QCOM_DMA_HEAP_H
|
||||
|
||||
#include <linux/bits.h>
|
||||
#include <linux/bitfield.h>
|
||||
#include <linux/dma-buf.h>
|
||||
|
||||
#include <linux/qcom_dma_heap_dt_constants.h>
|
||||
#include <linux/err.h>
|
||||
|
||||
/* Heap flags */
|
||||
#define QCOM_DMA_HEAP_FLAG_CACHED BIT(1)
|
||||
|
||||
#define QCOM_DMA_HEAP_FLAG_CP_TRUSTED_VM BIT(15)
|
||||
#define QCOM_DMA_HEAP_FLAG_CP_TZ BIT(16)
|
||||
#define QCOM_DMA_HEAP_FLAG_CP_TOUCH BIT(17)
|
||||
#define QCOM_DMA_HEAP_FLAG_CP_BITSTREAM BIT(18)
|
||||
#define QCOM_DMA_HEAP_FLAG_CP_PIXEL BIT(19)
|
||||
#define QCOM_DMA_HEAP_FLAG_CP_NON_PIXEL BIT(20)
|
||||
#define QCOM_DMA_HEAP_FLAG_CP_CAMERA BIT(21)
|
||||
#define QCOM_DMA_HEAP_FLAG_CP_HLOS BIT(22)
|
||||
#define QCOM_DMA_HEAP_FLAG_CP_SPSS_SP BIT(23)
|
||||
#define QCOM_DMA_HEAP_FLAG_CP_SPSS_SP_SHARED BIT(24)
|
||||
#define QCOM_DMA_HEAP_FLAG_CP_SEC_DISPLAY BIT(25)
|
||||
#define QCOM_DMA_HEAP_FLAG_CP_APP BIT(26)
|
||||
#define QCOM_DMA_HEAP_FLAG_CP_CAMERA_PREVIEW BIT(27)
|
||||
#define QCOM_DMA_HEAP_FLAG_CP_MSS_MSA BIT(28)
|
||||
#define QCOM_DMA_HEAP_FLAG_CP_CDSP BIT(29)
|
||||
#define QCOM_DMA_HEAP_FLAG_CP_SPSS_HLOS_SHARED BIT(30)
|
||||
|
||||
#define QCOM_DMA_HEAP_FLAGS_CP_MASK GENMASK(30, 15)
|
||||
|
||||
#define QCOM_DMA_HEAP_FLAG_SECURE BIT(31)
|
||||
|
||||
/**
|
||||
* carveout_heap_add_memory - Dynamically add reserved heap memory
|
||||
* @heap_name: The name of the DMA-BUF Heap we're giving memory to
|
||||
* @sgt: An SGT corresponding to the memory adding to the heap
|
||||
* @cookie: Cookie for managing refcount.
|
||||
* @get: Function callback for increasing refcount.
|
||||
* @put: Function callback for decreasing refcount.
|
||||
*
|
||||
* Return: 0 on success, a negative error value otherwise.
|
||||
*/
|
||||
int carveout_heap_add_memory(char *heap_name, struct sg_table *sgt, void *cookie,
|
||||
int (*get)(void *), void (*put)(void *));
|
||||
|
||||
|
||||
/**
|
||||
* carveout_heap_remove_memory - Dynamically remove reserved heap memory
|
||||
* @heap_name: The name of the DMA-BUF Heap we're taking memory away from
|
||||
* @sgt: An SGT corresponding to the memory we're removing from the heap
|
||||
*
|
||||
* Return: 0 on success, a negative error value otherwise.
|
||||
*/
|
||||
int carveout_heap_remove_memory(char *heap_name, struct sg_table *sgt);
|
||||
|
||||
struct dma_buf_heap_prefetch_region {
|
||||
u64 size;
|
||||
struct dma_heap *heap;
|
||||
};
|
||||
|
||||
int qcom_secure_system_heap_prefetch(struct dma_buf_heap_prefetch_region *regions,
|
||||
size_t nr_regions);
|
||||
|
||||
int qcom_secure_system_heap_drain(struct dma_buf_heap_prefetch_region *regions,
|
||||
size_t nr_regions);
|
||||
|
||||
/**
|
||||
* dma_buf_heap_hyp_assign - wrapper function for hyp-assigning a dma_buf
|
||||
* @buf: dma_buf to hyp-assign away from HLOS
|
||||
* @dest_vmids: array of QCOM_DMA_HEAP_FLAG VMIDs (as defined above)
|
||||
* @dest_perms: array of PERM_READ/PERM_WRITE/PERM_EXEC permission bits (as
|
||||
* defined in include/soc/qcom/secure_buffer.h), such that
|
||||
* dest_perms[i] specifies the permissions for VMID dest_vmids[i]
|
||||
* @dest_nelems: number of elements in dest_vmids and dest_perms
|
||||
*
|
||||
* Return: Temporarily return -EINVAL whilst the functon isn't present. Otherwise, return
|
||||
* 0 on success, or a negative value returned by hyp_assign_table() on failure.
|
||||
*/
|
||||
static inline int dma_buf_heap_hyp_assign(struct dma_buf *buf, int *dest_vmids, int *dest_perms,
|
||||
int dest_nelems)
|
||||
{
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
/**
|
||||
* dma_buf_heap_hyp_unassign - wrapper function that hyp-assigns a dma_buf back to HLOS
|
||||
* @buf: dma_buf to hyp-assign back to HLOS
|
||||
*
|
||||
* This function takes a dma_buf, and re-assigns it to HLOS with RWX permissions (at the
|
||||
* S2 level). The end-points to which the buffer was assigned to are tracked by the flags
|
||||
* kept in the msm_heap_helper_buffer->flags field (the helper_buffer is accesed through
|
||||
* dma_buf->priv), so the corresponding VMIDs don't need to be supplied as arguments.
|
||||
*
|
||||
* Return: Temporarily return -EINVAL whilst the functon isn't present. Otherwise, return
|
||||
* 0 on success, or a negative value returned by hyp_assign_table() on failure.
|
||||
*/
|
||||
static inline int dma_buf_heap_hyp_unassign(struct dma_buf *buf)
|
||||
{
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
#endif /* _QCOM_DMA_HEAP_H */
|
||||
12
include/linux/qcom_dma_heap_dt_constants.h
Normal file
12
include/linux/qcom_dma_heap_dt_constants.h
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
#ifndef _QCOM_DMA_HEAP_DT_CONSTATS_H
|
||||
#define _QCOM_DMA_HEAP_DT_CONSTATS_H
|
||||
|
||||
#define HEAP_TYPE_SECURE_CARVEOUT 0
|
||||
#define HEAP_TYPE_CARVEOUT 1
|
||||
#define HEAP_TYPE_CMA 2
|
||||
|
||||
#endif /* _QCOM_DMA_HEAP_DT_CONSTATS_H */
|
||||
95
include/uapi/linux/msm_ion.h
Normal file
95
include/uapi/linux/msm_ion.h
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
|
||||
/*
|
||||
* Copyright (c) 2018-2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
#ifndef _UAPI_LINUX_MSM_ION_H
|
||||
#define _UAPI_LINUX_MSM_ION_H
|
||||
|
||||
#include <linux/types.h>
|
||||
#include <linux/msm_ion_ids.h>
|
||||
|
||||
/**
|
||||
* TARGET_ION_ABI_VERSION can be used by user space clients to ensure that at
|
||||
* compile time only their code which uses the appropriate ION APIs for
|
||||
* this kernel is included.
|
||||
*/
|
||||
#define TARGET_ION_ABI_VERSION 3
|
||||
|
||||
enum msm_ion_heap_types {
|
||||
ION_HEAP_TYPE_MSM_START = 16,
|
||||
ION_HEAP_TYPE_SECURE_DMA = ION_HEAP_TYPE_MSM_START,
|
||||
ION_HEAP_TYPE_SYSTEM_SECURE,
|
||||
ION_HEAP_TYPE_HYP_CMA,
|
||||
ION_HEAP_TYPE_MSM_CARVEOUT,
|
||||
ION_HEAP_TYPE_SECURE_CARVEOUT,
|
||||
ION_HEAP_TYPE_MSM_SYSTEM,
|
||||
};
|
||||
|
||||
/**
|
||||
* Flags to be used when allocating from the secure heap for
|
||||
* content protection
|
||||
*/
|
||||
#define ION_FLAG_CP_TRUSTED_VM ION_BIT(15)
|
||||
/* ION_FLAG_POOL_FORCE_ALLOC uses ION_BIT(16) */
|
||||
#define ION_FLAG_CP_TOUCH ION_BIT(17)
|
||||
#define ION_FLAG_CP_BITSTREAM ION_BIT(18)
|
||||
#define ION_FLAG_CP_PIXEL ION_BIT(19)
|
||||
#define ION_FLAG_CP_NON_PIXEL ION_BIT(20)
|
||||
#define ION_FLAG_CP_CAMERA ION_BIT(21)
|
||||
#define ION_FLAG_CP_HLOS ION_BIT(22)
|
||||
#define ION_FLAG_CP_SPSS_SP ION_BIT(23)
|
||||
#define ION_FLAG_CP_SPSS_SP_SHARED ION_BIT(24)
|
||||
#define ION_FLAG_CP_SEC_DISPLAY ION_BIT(25)
|
||||
#define ION_FLAG_CP_APP ION_BIT(26)
|
||||
#define ION_FLAG_CP_CAMERA_PREVIEW ION_BIT(27)
|
||||
/* ION_FLAG_ALLOW_NON_CONTIG uses ION_BIT(28) */
|
||||
#define ION_FLAG_CP_CDSP ION_BIT(29)
|
||||
#define ION_FLAG_CP_SPSS_HLOS_SHARED ION_BIT(30)
|
||||
|
||||
#define ION_FLAGS_CP_MASK 0x6FFE8000
|
||||
|
||||
/**
|
||||
* Flag to allow non continguous allocation of memory from secure
|
||||
* heap
|
||||
*/
|
||||
#define ION_FLAG_ALLOW_NON_CONTIG ION_BIT(28)
|
||||
|
||||
/**
|
||||
* Flag to use when allocating to indicate that a heap is secure.
|
||||
* Do NOT use BIT macro since it is defined in #ifdef __KERNEL__
|
||||
*/
|
||||
#define ION_FLAG_SECURE ION_BIT(31)
|
||||
|
||||
/*
|
||||
* Used in conjunction with heap which pool memory to force an allocation
|
||||
* to come from the page allocator directly instead of from the pool allocation
|
||||
*/
|
||||
#define ION_FLAG_POOL_FORCE_ALLOC ION_BIT(16)
|
||||
|
||||
/**
|
||||
* Macro should be used with ion_heap_ids defined above.
|
||||
*/
|
||||
#define ION_HEAP(bit) bit
|
||||
|
||||
#define ION_IOC_MSM_MAGIC 'M'
|
||||
|
||||
struct ion_prefetch_regions {
|
||||
__u64 sizes;
|
||||
__u32 vmid;
|
||||
__u32 nr_sizes;
|
||||
};
|
||||
|
||||
struct ion_prefetch_data {
|
||||
__u64 len;
|
||||
__u64 regions;
|
||||
__u32 heap_id;
|
||||
__u32 nr_regions;
|
||||
};
|
||||
|
||||
#define ION_IOC_PREFETCH _IOWR(ION_IOC_MSM_MAGIC, 3, \
|
||||
struct ion_prefetch_data)
|
||||
|
||||
#define ION_IOC_DRAIN _IOWR(ION_IOC_MSM_MAGIC, 4, \
|
||||
struct ion_prefetch_data)
|
||||
|
||||
#endif /* _UAPI_LINUX_MSM_ION_H */
|
||||
37
include/uapi/linux/msm_ion_ids.h
Normal file
37
include/uapi/linux/msm_ion_ids.h
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only WITH Linux-syscall-note */
|
||||
/*
|
||||
* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
#ifndef _MSM_ION_IDS_H
|
||||
#define _MSM_ION_IDS_H
|
||||
|
||||
#define ION_BIT(nr) (1U << (nr))
|
||||
|
||||
/**
|
||||
* These are the only ids that should be used for Ion heap ids.
|
||||
* The ids listed are the order in which allocation will be attempted
|
||||
* if specified. Don't swap the order of heap ids unless you know what
|
||||
* you are doing!
|
||||
* Id's are spaced by purpose to allow new Id's to be inserted in-between (for
|
||||
* possible fallbacks)
|
||||
*/
|
||||
|
||||
/* ION_BIT(0) is reserved for the generic system heap. */
|
||||
#define ION_QSECOM_TA_HEAP_ID ION_BIT(1)
|
||||
#define ION_CAMERA_HEAP_ID ION_BIT(2)
|
||||
#define ION_DISPLAY_HEAP_ID ION_BIT(3)
|
||||
#define ION_ADSP_HEAP_ID ION_BIT(4)
|
||||
#define ION_AUDIO_ML_HEAP_ID ION_BIT(5)
|
||||
#define ION_USER_CONTIG_HEAP_ID ION_BIT(6)
|
||||
#define ION_QSECOM_HEAP_ID ION_BIT(7)
|
||||
#define ION_AUDIO_HEAP_ID ION_BIT(8)
|
||||
#define ION_CP_MM_HEAP_ID ION_BIT(9)
|
||||
#define ION_SECURE_HEAP_ID ION_BIT(10)
|
||||
#define ION_SECURE_DISPLAY_HEAP_ID ION_BIT(11)
|
||||
#define ION_SPSS_HEAP_ID ION_BIT(14)
|
||||
#define ION_SECURE_CARVEOUT_HEAP_ID ION_BIT(15)
|
||||
#define ION_TUI_CARVEOUT_HEAP_ID ION_BIT(16)
|
||||
#define ION_SYSTEM_HEAP_ID ION_BIT(25)
|
||||
#define ION_HEAP_ID_RESERVED ION_BIT(31)
|
||||
|
||||
#endif /* _MSM_ION_IDS_H */
|
||||
Loading…
Reference in New Issue
Block a user