From 76b314bf2d3f83b1a1a46a8d7d33657c3b058199 Mon Sep 17 00:00:00 2001 From: Chris Goldsworthy Date: Thu, 16 Jun 2022 12:11:55 -0700 Subject: [PATCH 1/2] mem-buf: Take a snapshot of the mem-buf driver from msm-5.15 Take a snapsnot of the mem-buf driver as of commit 49c19cd9f512 ("Merge "defconfig: kalama: Enable cfg80211 and mac80211 as modules"") on msm-5.15. Change-Id: I8400bc0e22dc7af2dac34c9d4859e30964e87c90 Signed-off-by: Chris Goldsworthy --- drivers/soc/qcom/Kconfig | 2 + drivers/soc/qcom/Makefile | 1 + drivers/soc/qcom/mem_buf/Kconfig | 39 + drivers/soc/qcom/mem_buf/Makefile | 9 + drivers/soc/qcom/mem_buf/mem-buf-dev-gh.c | 422 ++++++ drivers/soc/qcom/mem_buf/mem-buf-dev.c | 170 +++ drivers/soc/qcom/mem_buf/mem-buf-dev.h | 109 ++ drivers/soc/qcom/mem_buf/mem-buf-gh.c | 1404 ++++++++++++++++++++ drivers/soc/qcom/mem_buf/mem-buf-gh.h | 54 + drivers/soc/qcom/mem_buf/mem-buf-ids.c | 340 +++++ drivers/soc/qcom/mem_buf/mem-buf-ids.h | 46 + drivers/soc/qcom/mem_buf/mem-buf-msgq.c | 453 +++++++ drivers/soc/qcom/mem_buf/mem-buf-msgq.h | 257 ++++ drivers/soc/qcom/mem_buf/mem-buf.c | 392 ++++++ drivers/soc/qcom/mem_buf/mem_buf_dma_buf.c | 681 ++++++++++ drivers/soc/qcom/mem_buf/trace-mem-buf.h | 299 +++++ 16 files changed, 4678 insertions(+) create mode 100644 drivers/soc/qcom/mem_buf/Kconfig create mode 100644 drivers/soc/qcom/mem_buf/Makefile create mode 100644 drivers/soc/qcom/mem_buf/mem-buf-dev-gh.c create mode 100644 drivers/soc/qcom/mem_buf/mem-buf-dev.c create mode 100644 drivers/soc/qcom/mem_buf/mem-buf-dev.h create mode 100644 drivers/soc/qcom/mem_buf/mem-buf-gh.c create mode 100644 drivers/soc/qcom/mem_buf/mem-buf-gh.h create mode 100644 drivers/soc/qcom/mem_buf/mem-buf-ids.c create mode 100644 drivers/soc/qcom/mem_buf/mem-buf-ids.h create mode 100644 drivers/soc/qcom/mem_buf/mem-buf-msgq.c create mode 100644 drivers/soc/qcom/mem_buf/mem-buf-msgq.h create mode 100644 drivers/soc/qcom/mem_buf/mem-buf.c create mode 100644 drivers/soc/qcom/mem_buf/mem_buf_dma_buf.c create mode 100644 drivers/soc/qcom/mem_buf/trace-mem-buf.h diff --git a/drivers/soc/qcom/Kconfig b/drivers/soc/qcom/Kconfig index c847e2048547..3faa993405df 100644 --- a/drivers/soc/qcom/Kconfig +++ b/drivers/soc/qcom/Kconfig @@ -562,4 +562,6 @@ config MINIDUMP_MAX_ENTRIES This defines maximum number of entries to be allocated for application subsytem in Minidump table. +source "drivers/soc/qcom/mem_buf/Kconfig" + endmenu diff --git a/drivers/soc/qcom/Makefile b/drivers/soc/qcom/Makefile index 3eee74fe0ca2..53b2c0839219 100644 --- a/drivers/soc/qcom/Makefile +++ b/drivers/soc/qcom/Makefile @@ -24,6 +24,7 @@ obj-$(CONFIG_QTI_PMIC_GLINK) += pmic_glink.o obj-$(CONFIG_QTI_BATTERY_GLINK_DEBUG) += qti_battery_debug.o obj-$(CONFIG_QTI_CHARGER_ULOG_GLINK) += charger-ulog-glink.o obj-$(CONFIG_QTI_ALTMODE_GLINK) += altmode-glink.o +obj-$(CONFIG_QCOM_MEM_BUF) += mem_buf/ obj-$(CONFIG_QCOM_SECURE_BUFFER) += secure_buffer.o obj-$(CONFIG_QCOM_SOCINFO) += socinfo.o obj-$(CONFIG_QCOM_SPM) += spm.o diff --git a/drivers/soc/qcom/mem_buf/Kconfig b/drivers/soc/qcom/mem_buf/Kconfig new file mode 100644 index 000000000000..273d1a77dd1a --- /dev/null +++ b/drivers/soc/qcom/mem_buf/Kconfig @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config QCOM_MEM_BUF + tristate "Qualcomm Technologies, Inc. Memory Buffer Sharing Driver" + select QCOM_MEM_BUF_DEV + help + Add support for lending memory from one virtual machine to another. + This driver communicates with the hypervisor, as well as other + virtual machines, to request and lend memory from and to VMs + respectively. + If unsure, say N + +config QCOM_MEM_BUF_GH + bool "Qualcomm Technologies, Inc. Memory Buffer Gunyah Hypervisor Support" + depends on GH_RM_DRV + select QCOM_MEM_BUF_DEV_GH + help + Enables additional support for sharing memory with VMs that have their own + operating systems, as opposed to peripheral VMs, which do not have their + own operating systems. These VMs are managed by the Gunyah Hypervisor, and thus + sharing memory with them requires interacting with the Gunyah Hypervisor. + If unsure, say N. + +config QCOM_MEM_BUF_DEV + tristate + +config QCOM_MEM_BUF_DEV_GH + bool + +config QCOM_MEM_BUF_MSGQ + tristate "Qualcomm Technologies, Inc. Memory Buffer Message Queue Support" + depends on GH_MSGQ && QCOM_MEM_BUF_DEV_GH + help + Enables support for communicating with VMs over message queues. The communication + with the VMs follows a specific communication format that is defined by the mem-buf + message queue protocol. The protocol allows for allocation requests to be submitted + between VMs, and relinquished when a VM is finished with the memory that it + requested from another VM. + If unsure, say N. diff --git a/drivers/soc/qcom/mem_buf/Makefile b/drivers/soc/qcom/mem_buf/Makefile new file mode 100644 index 000000000000..eaf7733dd3b3 --- /dev/null +++ b/drivers/soc/qcom/mem_buf/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_QCOM_MEM_BUF) += mem_buf.o +mem_buf-y += mem-buf.o +mem_buf-$(CONFIG_QCOM_MEM_BUF_GH) += mem-buf-gh.o +obj-$(CONFIG_QCOM_MEM_BUF_DEV) += mem_buf_dev.o +mem_buf_dev-y += mem-buf-dev.o mem_buf_dma_buf.o mem-buf-ids.o +mem_buf_dev-$(CONFIG_QCOM_MEM_BUF_DEV_GH) += mem-buf-dev-gh.o +obj-$(CONFIG_QCOM_MEM_BUF_MSGQ) += mem_buf_msgq.o +mem_buf_msgq-y += mem-buf-msgq.o diff --git a/drivers/soc/qcom/mem_buf/mem-buf-dev-gh.c b/drivers/soc/qcom/mem_buf/mem-buf-dev-gh.c new file mode 100644 index 000000000000..18194d375677 --- /dev/null +++ b/drivers/soc/qcom/mem_buf/mem-buf-dev-gh.c @@ -0,0 +1,422 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include +#include + +#include + +#include "mem-buf-dev.h" +#include "mem-buf-ids.h" + +#define CREATE_TRACE_POINTS +#include "trace-mem-buf.h" +EXPORT_TRACEPOINT_SYMBOL(send_alloc_req); +EXPORT_TRACEPOINT_SYMBOL(receive_alloc_req); +EXPORT_TRACEPOINT_SYMBOL(send_relinquish_msg); +EXPORT_TRACEPOINT_SYMBOL(receive_relinquish_msg); +EXPORT_TRACEPOINT_SYMBOL(send_alloc_resp_msg); +EXPORT_TRACEPOINT_SYMBOL(receive_alloc_resp_msg); +EXPORT_TRACEPOINT_SYMBOL(mem_buf_alloc_info); + +struct gh_acl_desc *mem_buf_vmid_perm_list_to_gh_acl(int *vmids, int *perms, + unsigned int nr_acl_entries) +{ + struct gh_acl_desc *gh_acl; + size_t size; + unsigned int i; + + size = offsetof(struct gh_acl_desc, acl_entries[nr_acl_entries]); + gh_acl = kmalloc(size, GFP_KERNEL); + if (!gh_acl) + return ERR_PTR(-ENOMEM); + + gh_acl->n_acl_entries = nr_acl_entries; + for (i = 0; i < nr_acl_entries; i++) { + gh_acl->acl_entries[i].vmid = vmids[i]; + gh_acl->acl_entries[i].perms = perms[i]; + } + + return gh_acl; +} +EXPORT_SYMBOL(mem_buf_vmid_perm_list_to_gh_acl); + +struct gh_sgl_desc *mem_buf_sgt_to_gh_sgl_desc(struct sg_table *sgt) +{ + struct gh_sgl_desc *gh_sgl; + size_t size; + int i; + struct scatterlist *sg; + + size = offsetof(struct gh_sgl_desc, sgl_entries[sgt->orig_nents]); + gh_sgl = kvmalloc(size, GFP_KERNEL); + if (!gh_sgl) + return ERR_PTR(-ENOMEM); + + gh_sgl->n_sgl_entries = sgt->orig_nents; + for_each_sgtable_sg(sgt, sg, i) { + gh_sgl->sgl_entries[i].ipa_base = sg_phys(sg); + gh_sgl->sgl_entries[i].size = sg->length; + } + + return gh_sgl; +} +EXPORT_SYMBOL(mem_buf_sgt_to_gh_sgl_desc); + +int mem_buf_gh_acl_desc_to_vmid_perm_list(struct gh_acl_desc *acl_desc, + int **vmids, int **perms) +{ + int *vmids_arr = NULL, *perms_arr = NULL; + u32 nr_acl_entries = acl_desc->n_acl_entries; + unsigned int i; + + if (!vmids || !perms) + return -EINVAL; + + vmids_arr = kmalloc_array(nr_acl_entries, sizeof(*vmids_arr), + GFP_KERNEL); + if (!vmids_arr) + return -ENOMEM; + + perms_arr = kmalloc_array(nr_acl_entries, sizeof(*perms_arr), + GFP_KERNEL); + if (!perms_arr) { + kfree(vmids_arr); + return -ENOMEM; + } + + *vmids = vmids_arr; + *perms = perms_arr; + + for (i = 0; i < nr_acl_entries; i++) { + vmids_arr[i] = acl_desc->acl_entries[i].vmid; + perms_arr[i] = acl_desc->acl_entries[i].perms; + } + + return 0; +} +EXPORT_SYMBOL(mem_buf_gh_acl_desc_to_vmid_perm_list); + +struct sg_table *dup_gh_sgl_desc_to_sgt(struct gh_sgl_desc *sgl_desc) +{ + struct sg_table *new_table; + int ret, i; + struct scatterlist *sg; + + if (!sgl_desc || !sgl_desc->n_sgl_entries) + return ERR_PTR(-EINVAL); + + new_table = kzalloc(sizeof(*new_table), GFP_KERNEL); + if (!new_table) + return ERR_PTR(-ENOMEM); + + ret = sg_alloc_table(new_table, sgl_desc->n_sgl_entries, GFP_KERNEL); + if (ret) { + kfree(new_table); + return ERR_PTR(-ENOMEM); + } + + for_each_sg(new_table->sgl, sg, new_table->nents, i) { + sg_set_page(sg, phys_to_page(sgl_desc->sgl_entries[i].ipa_base), + sgl_desc->sgl_entries[i].size, 0); + sg_dma_address(sg) = 0; + sg_dma_len(sg) = 0; + } + + return new_table; +} +EXPORT_SYMBOL(dup_gh_sgl_desc_to_sgt); + +size_t mem_buf_get_sgl_buf_size(struct gh_sgl_desc *sgl_desc) +{ + size_t size = 0; + unsigned int i; + + for (i = 0; i < sgl_desc->n_sgl_entries; i++) + size += sgl_desc->sgl_entries[i].size; + + return size; +} +EXPORT_SYMBOL(mem_buf_get_sgl_buf_size); + +static int __mem_buf_map_mem_s2_cleanup_donate(struct gh_sgl_desc *sgl_desc, + int src_vmid, gh_memparcel_handle_t *handle) +{ + int ret; + int src_perms = PERM_READ | PERM_WRITE | PERM_EXEC; + struct mem_buf_lend_kernel_arg arg = { + .nr_acl_entries = 1, + .vmids = &src_vmid, + .perms = &src_perms, + .flags = 0, //No sanitize as buffer unmodified. + .label = 0, + }; + struct sg_table *sgt; + + sgt = dup_gh_sgl_desc_to_sgt(sgl_desc); + if (IS_ERR(sgt)) + return PTR_ERR(sgt); + + ret = mem_buf_assign_mem_gunyah(GH_RM_TRANS_TYPE_DONATE, sgt, &arg); + if (!ret) + *handle = arg.memparcel_hdl; + + sg_free_table(sgt); + kfree(sgt); + return ret; +} + +static int mem_buf_hyp_assign_table_gh(struct gh_sgl_desc *sgl_desc, int src_vmid, + struct gh_acl_desc *acl_desc); + +/* + * @memparcel_hdl: + * GH_RM_TRANS_TYPE_DONATE - memparcel_hdl will be set to MEM_BUF_MEMPARCEL_INVALID + on success, and (possibly) set to a different valid memparcel on error. This is + because accepting a donated memparcel handle destroys that handle. + GH_RM_TRANS_TYPE_LEND - unmodified. + GH_RM_TRANS_TYPE_SHARE - unmodified. + */ +struct gh_sgl_desc *mem_buf_map_mem_s2(int op, gh_memparcel_handle_t *__memparcel_hdl, + struct gh_acl_desc *acl_desc, int src_vmid) +{ + int ret, ret2; + struct gh_sgl_desc *sgl_desc; + u8 flags = GH_RM_MEM_ACCEPT_VALIDATE_ACL_ATTRS | + GH_RM_MEM_ACCEPT_DONE; + gh_memparcel_handle_t memparcel_hdl = *__memparcel_hdl; + + if (!acl_desc) + return ERR_PTR(-EINVAL); + + /* + * memory returns to its original IPA address when accepted by HLOS. For example, + * scattered memory returns to being scattered memory. + */ + if (current_vmid != VMID_HLOS) + flags |= GH_RM_MEM_ACCEPT_MAP_IPA_CONTIGUOUS; + + pr_debug("%s: adding CPU MMU stage 2 mappings\n", __func__); + sgl_desc = gh_rm_mem_accept(memparcel_hdl, GH_RM_MEM_TYPE_NORMAL, op, + flags, 0, acl_desc, NULL, + NULL, 0); + if (IS_ERR(sgl_desc)) { + pr_err("%s failed to map memory in stage 2 rc: %d\n", __func__, + PTR_ERR(sgl_desc)); + return sgl_desc; + } + + if (op == GH_RM_TRANS_TYPE_DONATE) + *__memparcel_hdl = MEM_BUF_MEMPARCEL_INVALID; + + ret = mem_buf_hyp_assign_table_gh(sgl_desc, src_vmid, acl_desc); + if (ret) + goto err_relinquish; + + trace_map_mem_s2(memparcel_hdl, sgl_desc); + return sgl_desc; + +err_relinquish: + if (op == GH_RM_TRANS_TYPE_DONATE) + ret2 = __mem_buf_map_mem_s2_cleanup_donate(sgl_desc, src_vmid, + __memparcel_hdl); + else + ret2 = mem_buf_unmap_mem_s2(memparcel_hdl); + kvfree(sgl_desc); + if (ret2) { + pr_err("%s failed to recover\n", __func__); + return ERR_PTR(-EADDRNOTAVAIL); + } + return ERR_PTR(ret); +} +EXPORT_SYMBOL(mem_buf_map_mem_s2); + +int mem_buf_unmap_mem_s2(gh_memparcel_handle_t memparcel_hdl) +{ + int ret; + + pr_debug("%s: removing CPU MMU stage 2 mappings\n", __func__); + ret = gh_rm_mem_release(memparcel_hdl, 0); + + if (ret < 0) + pr_err("%s: Failed to release memparcel hdl: 0x%lx rc: %d\n", + __func__, memparcel_hdl, ret); + else + pr_debug("%s: CPU MMU stage 2 mappings removed\n", __func__); + + return ret; +} +EXPORT_SYMBOL(mem_buf_unmap_mem_s2); + +int mem_buf_map_mem_s1(struct gh_sgl_desc *sgl_desc) +{ + u64 base, size; + int i, ret; + + for (i = 0; i < sgl_desc->n_sgl_entries; i++) { + base = sgl_desc->sgl_entries[i].ipa_base; + size = sgl_desc->sgl_entries[i].size; + + ret = add_memory_subsection(numa_node_id(), base, size); + if (ret) { + pr_err("%s: failed to add memory base=%llx, size=%llx, ret=%d\n", + __func__, base, size, ret); + goto out; + } + } + + return 0; + +out: + for (i--; i >= 0; i--) { + base = sgl_desc->sgl_entries[i].ipa_base; + size = sgl_desc->sgl_entries[i].size; + remove_memory_subsection(base, size); + } + + return ret; +} +EXPORT_SYMBOL(mem_buf_map_mem_s1); + +int mem_buf_unmap_mem_s1(struct gh_sgl_desc *sgl_desc) +{ + u64 base, size; + int i, ret; + + for (i = 0; i < sgl_desc->n_sgl_entries; i++) { + base = sgl_desc->sgl_entries[i].ipa_base; + size = sgl_desc->sgl_entries[i].size; + + ret = remove_memory_subsection(base, size); + if (ret) + pr_err("%s: failed to remove memory base=%llx, size=%llx\n, ret=%d\n", + __func__, base, size, ret); + } + + return ret; +} +EXPORT_SYMBOL(mem_buf_unmap_mem_s1); + +static int mem_buf_hyp_assign_table_gh(struct gh_sgl_desc *sgl_desc, int src_vmid, + struct gh_acl_desc *acl_desc) +{ + struct sg_table *sgt; + int *dst_vmids, *dst_perms; + int ret; + + sgt = dup_gh_sgl_desc_to_sgt(sgl_desc); + if (IS_ERR(sgt)) + return PTR_ERR(sgt); + + ret = mem_buf_gh_acl_desc_to_vmid_perm_list(acl_desc, &dst_vmids, &dst_perms); + if (ret) + goto err_free_sgt; + + ret = mem_buf_hyp_assign_table(sgt, &src_vmid, 1, dst_vmids, dst_perms, + acl_desc->n_acl_entries); + kfree(dst_vmids); + kfree(dst_perms); +err_free_sgt: + sg_free_table(sgt); + kfree(sgt); + return ret; +} + +int mem_buf_assign_mem_gunyah(int op, struct sg_table *sgt, + struct mem_buf_lend_kernel_arg *arg) +{ + int ret, i; + struct gh_sgl_desc *gh_sgl; + struct gh_acl_desc *gh_acl; + size_t size; + struct scatterlist *sgl; + + arg->memparcel_hdl = MEM_BUF_MEMPARCEL_INVALID; + ret = mem_buf_vm_uses_gunyah(arg->vmids, arg->nr_acl_entries); + if (ret <= 0) + return ret; + + /* Physically contiguous memory only */ + if (sgt->nents > 1) { + pr_err_ratelimited("Operation requires physically contiguous memory\n"); + return -EINVAL; + } + + /* Due to memory-hotplug */ + size = 0; + for_each_sgtable_sg(sgt, sgl, i) + size += sgl->length; + if (!IS_ALIGNED(size, SUBSECTION_SIZE)) { + pr_err_ratelimited("Operation requires SUBSECTION_SIZE alignemnt, size = %zx\n", + size); + return -EINVAL; + } + + gh_sgl = mem_buf_sgt_to_gh_sgl_desc(sgt); + if (IS_ERR(gh_sgl)) + return PTR_ERR(gh_sgl); + + gh_acl = mem_buf_vmid_perm_list_to_gh_acl(arg->vmids, arg->perms, + arg->nr_acl_entries); + if (IS_ERR(gh_acl)) { + ret = PTR_ERR(gh_acl); + goto err_gh_acl; + } + + pr_debug("%s: Invoking Gunyah Lend/Share\n", __func__); + if (op == GH_RM_TRANS_TYPE_LEND) { + ret = gh_rm_mem_lend(GH_RM_MEM_TYPE_NORMAL, arg->flags, + arg->label, gh_acl, gh_sgl, + NULL /* Default memory attributes */, + &arg->memparcel_hdl); + } else if (op == GH_RM_TRANS_TYPE_SHARE) { + ret = gh_rm_mem_share(GH_RM_MEM_TYPE_NORMAL, arg->flags, + arg->label, gh_acl, gh_sgl, + NULL /* Default memory attributes */, + &arg->memparcel_hdl); + } else if (op == GH_RM_TRANS_TYPE_DONATE) { + ret = gh_rm_mem_donate(GH_RM_MEM_TYPE_NORMAL, arg->flags, + arg->label, gh_acl, gh_sgl, + NULL /* Default memory attributes */, + &arg->memparcel_hdl); + } else { + pr_err("%s: Unrecognized op %d\n", op); + ret = -EINVAL; + } + + if (ret < 0) { + pr_err("%s: Gunyah lend/share failed rc:%d\n", + __func__, ret); + goto err_gunyah; + } + + kfree(gh_acl); + kvfree(gh_sgl); + return 0; + +err_gunyah: + kfree(gh_acl); +err_gh_acl: + kvfree(gh_sgl); + + return ret; +} + +int mem_buf_unassign_mem_gunyah(gh_memparcel_handle_t memparcel_hdl) +{ + int ret; + + pr_debug("%s: Beginning gunyah reclaim\n", __func__); + ret = gh_rm_mem_reclaim(memparcel_hdl, 0); + if (ret) { + pr_err("%s: Gunyah reclaim failed\n", __func__); + return ret; + } + pr_debug("%s: Finished gunyah reclaim\n", __func__); + + return ret; +} diff --git a/drivers/soc/qcom/mem_buf/mem-buf-dev.c b/drivers/soc/qcom/mem_buf/mem-buf-dev.c new file mode 100644 index 000000000000..c32c00fbcdb3 --- /dev/null +++ b/drivers/soc/qcom/mem_buf/mem-buf-dev.c @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include "mem-buf-dev.h" +#include "mem-buf-ids.h" + +struct device *mem_buf_dev; +EXPORT_SYMBOL(mem_buf_dev); + +unsigned char mem_buf_capability; +EXPORT_SYMBOL(mem_buf_capability); + +int mem_buf_hyp_assign_table(struct sg_table *sgt, u32 *src_vmid, int source_nelems, + int *dest_vmids, int *dest_perms, int dest_nelems) +{ + char *verb; + int ret; + + if (!mem_buf_vm_uses_hyp_assign()) + return 0; + + verb = *src_vmid == current_vmid ? "Assign" : "Unassign"; + + pr_debug("%s memory to target VMIDs\n", verb); + ret = hyp_assign_table(sgt, src_vmid, source_nelems, dest_vmids, dest_perms, dest_nelems); + if (ret < 0) + pr_err("Failed to %s memory for rmt allocation rc: %d\n", verb, ret); + else + pr_debug("Memory %s to target VMIDs\n", verb); + + return ret; +} + +int mem_buf_assign_mem(int op, struct sg_table *sgt, + struct mem_buf_lend_kernel_arg *arg) +{ + int src_vmid[] = {current_vmid}; + int src_perms[] = {PERM_READ | PERM_WRITE | PERM_EXEC}; + int ret, ret2; + + if (!sgt || !arg->nr_acl_entries || !arg->vmids || !arg->perms) + return -EINVAL; + + ret = mem_buf_hyp_assign_table(sgt, src_vmid, ARRAY_SIZE(src_vmid), arg->vmids, arg->perms, + arg->nr_acl_entries); + if (ret) + return ret; + + ret = mem_buf_assign_mem_gunyah(op, sgt, arg); + if (ret) { + ret2 = mem_buf_hyp_assign_table(sgt, arg->vmids, arg->nr_acl_entries, + src_vmid, src_perms, ARRAY_SIZE(src_vmid)); + if (ret2 < 0) { + pr_err("hyp_assign failed while recovering from another error: %d\n", + ret2); + return -EADDRNOTAVAIL; + } + } + + return ret; +} +EXPORT_SYMBOL(mem_buf_assign_mem); + +int mem_buf_unassign_mem(struct sg_table *sgt, int *src_vmids, + unsigned int nr_acl_entries, + gh_memparcel_handle_t memparcel_hdl) +{ + int dst_vmid[] = {current_vmid}; + int dst_perm[] = {PERM_READ | PERM_WRITE | PERM_EXEC}; + int ret; + + if (!sgt || !src_vmids || !nr_acl_entries) + return -EINVAL; + + if (memparcel_hdl != MEM_BUF_MEMPARCEL_INVALID) { + ret = mem_buf_unassign_mem_gunyah(memparcel_hdl); + if (ret) + return ret; + } + + ret = mem_buf_hyp_assign_table(sgt, src_vmids, nr_acl_entries, + dst_vmid, dst_perm, ARRAY_SIZE(dst_vmid)); + return ret; +} +EXPORT_SYMBOL(mem_buf_unassign_mem); + +static int mem_buf_probe(struct platform_device *pdev) +{ + int ret; + struct device *dev = &pdev->dev; + u64 dma_mask = IS_ENABLED(CONFIG_ARM64) ? DMA_BIT_MASK(64) : + DMA_BIT_MASK(32); + + if (of_property_match_string(dev->of_node, "qcom,mem-buf-capabilities", + "supplier") >= 0) + mem_buf_capability = MEM_BUF_CAP_SUPPLIER; + else if (of_property_match_string(dev->of_node, + "qcom,mem-buf-capabilities", + "consumer") >= 0) + mem_buf_capability = MEM_BUF_CAP_CONSUMER; + else if (of_property_match_string(dev->of_node, + "qcom,mem-buf-capabilities", + "dual") >= 0) + mem_buf_capability = MEM_BUF_CAP_DUAL; + else + mem_buf_capability = 0; + + ret = dma_set_mask_and_coherent(dev, dma_mask); + if (ret) { + dev_err(dev, "Unable to set dma mask: %d\n", ret); + return ret; + } + + ret = mem_buf_vm_init(dev); + if (ret) { + dev_err(dev, "mem_buf_vm_init failed %d\n", ret); + return ret; + } + + mem_buf_dev = dev; + return 0; +} + +static int mem_buf_remove(struct platform_device *pdev) +{ + mem_buf_dev = NULL; + return 0; +} + +static const struct of_device_id mem_buf_match_tbl[] = { + {.compatible = "qcom,mem-buf"}, + {}, +}; + +static struct platform_driver mem_buf_driver = { + .probe = mem_buf_probe, + .remove = mem_buf_remove, + .driver = { + .name = "mem-buf", + .of_match_table = of_match_ptr(mem_buf_match_tbl), + }, +}; + +static int __init mem_buf_dev_init(void) +{ + return platform_driver_register(&mem_buf_driver); +} +module_init(mem_buf_dev_init); + +static void __exit mem_buf_dev_exit(void) +{ + mem_buf_vm_exit(); + platform_driver_unregister(&mem_buf_driver); +} +module_exit(mem_buf_dev_exit); + +MODULE_DESCRIPTION("Qualcomm Technologies, Inc. Memory Buffer Sharing driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/soc/qcom/mem_buf/mem-buf-dev.h b/drivers/soc/qcom/mem_buf/mem-buf-dev.h new file mode 100644 index 000000000000..04a44dc6beca --- /dev/null +++ b/drivers/soc/qcom/mem_buf/mem-buf-dev.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef MEM_BUF_PRIVATE_H +#define MEM_BUF_PRIVATE_H + +#include +#include +#include +#include +#include + +#define MEM_BUF_MEMPARCEL_INVALID (U32_MAX) + +#define MEM_BUF_CAP_SUPPLIER BIT(0) +#define MEM_BUF_CAP_CONSUMER BIT(1) +#define MEM_BUF_CAP_DUAL (MEM_BUF_CAP_SUPPLIER | MEM_BUF_CAP_CONSUMER) +extern unsigned char mem_buf_capability; +extern struct device *mem_buf_dev; + +/* Hypervisor Interface */ +int mem_buf_assign_mem(int op, struct sg_table *sgt, + struct mem_buf_lend_kernel_arg *arg); +int mem_buf_unassign_mem(struct sg_table *sgt, int *src_vmids, + unsigned int nr_acl_entries, + gh_memparcel_handle_t hdl); +int mem_buf_hyp_assign_table(struct sg_table *sgt, u32 *src_vmid, int source_nelems, + int *dest_vmids, int *dest_perms, int dest_nelems); + +#ifdef CONFIG_QCOM_MEM_BUF_DEV_GH +int mem_buf_map_mem_s1(struct gh_sgl_desc *sgl_desc); +int mem_buf_unmap_mem_s1(struct gh_sgl_desc *sgl_desc); +struct gh_acl_desc *mem_buf_vmid_perm_list_to_gh_acl(int *vmids, int *perms, + unsigned int nr_acl_entries); +struct gh_sgl_desc *mem_buf_sgt_to_gh_sgl_desc(struct sg_table *sgt); +struct gh_sgl_desc *mem_buf_map_mem_s2(int op, gh_memparcel_handle_t *__memparcel_hdl, + struct gh_acl_desc *acl_desc, int src_vmid); +int mem_buf_unmap_mem_s2(gh_memparcel_handle_t memparcel_hdl); +int mem_buf_gh_acl_desc_to_vmid_perm_list(struct gh_acl_desc *acl_desc, + int **vmids, int **perms); +size_t mem_buf_get_sgl_buf_size(struct gh_sgl_desc *sgl_desc); +struct sg_table *dup_gh_sgl_desc_to_sgt(struct gh_sgl_desc *sgl_desc); +int mem_buf_assign_mem_gunyah(int op, struct sg_table *sgt, + struct mem_buf_lend_kernel_arg *arg); +int mem_buf_unassign_mem_gunyah(gh_memparcel_handle_t memparcel_hdl); +#else +static inline int mem_buf_map_mem_s1(struct gh_sgl_desc *sgl_desc) +{ + return -EINVAL; +} + +static inline int mem_buf_unmap_mem_s1(struct gh_sgl_desc *sgl_desc) +{ + return -EINVAL; +} + +static inline struct gh_acl_desc *mem_buf_vmid_perm_list_to_gh_acl(int *vmids, int *perms, + unsigned int nr_acl_entries) +{ + return ERR_PTR(-EINVAL); +} + +static inline struct gh_sgl_desc *mem_buf_sgt_to_gh_sgl_desc(struct sg_table *sgt) +{ + return ERR_PTR(-EINVAL); +} + +static inline struct gh_sgl_desc *mem_buf_map_mem_s2(int op, gh_memparcel_handle_t *__memparcel_hdl, + struct gh_acl_desc *acl_desc, int src_vmid) +{ + return ERR_PTR(-EINVAL); +} + +static inline int mem_buf_unmap_mem_s2(gh_memparcel_handle_t memparcel_hdl) +{ + return -EINVAL; +} + +static inline int mem_buf_gh_acl_desc_to_vmid_perm_list(struct gh_acl_desc *acl_desc, + int **vmids, int **perms) +{ + return -EINVAL; +} + +static inline size_t mem_buf_get_sgl_buf_size(struct gh_sgl_desc *sgl_desc) +{ + return 0; +} + +static inline struct sg_table *dup_gh_sgl_desc_to_sgt(struct gh_sgl_desc *sgl_desc) +{ + return ERR_PTR(-EINVAL); +} + +static inline int mem_buf_assign_mem_gunyah(int op, struct sg_table *sgt, + struct mem_buf_lend_kernel_arg *arg) +{ + return -EINVAL; +} + +static inline int mem_buf_unassign_mem_gunyah(gh_memparcel_handle_t memparcel_hdl) +{ + return -EINVAL; +} +#endif +#endif diff --git a/drivers/soc/qcom/mem_buf/mem-buf-gh.c b/drivers/soc/qcom/mem_buf/mem-buf-gh.c new file mode 100644 index 000000000000..46385955214f --- /dev/null +++ b/drivers/soc/qcom/mem_buf/mem-buf-gh.c @@ -0,0 +1,1404 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include + +#include "../../../../drivers/dma-buf/heaps/qcom_sg_ops.h" +#include "mem-buf-gh.h" +#include "mem-buf-msgq.h" +#include "mem-buf-ids.h" +#include "trace-mem-buf.h" + +#define MEM_BUF_MHP_ALIGNMENT (1UL << SUBSECTION_SHIFT) +#define MEM_BUF_TIMEOUT_MS 3500 +#define to_rmt_msg(_work) container_of(_work, struct mem_buf_rmt_msg, work) + +/* Maintains a list of memory buffers requested from other VMs */ +static DEFINE_MUTEX(mem_buf_list_lock); +static LIST_HEAD(mem_buf_list); + +/* Data structures for tracking message queue usage. */ +static struct workqueue_struct *mem_buf_wq; +static void *mem_buf_msgq_hdl; + +/* Maintains a list of memory buffers lent out to other VMs */ +static DEFINE_MUTEX(mem_buf_xfer_mem_list_lock); +static LIST_HEAD(mem_buf_xfer_mem_list); + +/** + * struct mem_buf_rmt_msg: Represents a message sent from a remote VM + * @msg: A pointer to the message buffer + * @msg_size: The size of the message + * @work: work structure for dispatching the message processing to a worker + * thread, so as to not block the message queue receiving thread. + */ +struct mem_buf_rmt_msg { + void *msg; + size_t msg_size; + struct work_struct work; +}; + +/** + * struct mem_buf_xfer_mem: Represents a memory buffer lent out or transferred + * to another VM. + * @size: The size of the memory buffer + * @mem_type: The type of memory that was allocated and transferred + * @mem_type_data: Data associated with the type of memory + * @mem_sgt: An SG-Table representing the memory transferred + * @secure_alloc: Denotes if the memory was assigned to the targeted VMs as part + * of the allocation step + * @hdl: The memparcel handle associated with the memory + * @gh_rm_trans_type: The type of memory transfer associated with the memory (donation, + * share, lend). + * @entry: List entry for maintaining a list of memory buffers that are lent + * out. + * @nr_acl_entries: The number of VMIDs and permissions associated with the + * memory + * @dst_vmids: The VMIDs that have access to the memory + * @dst_perms: The access permissions for the VMIDs that can access the memory + * @txn_id: The transaction ID that corresponds to this memory buffer + */ +struct mem_buf_xfer_mem { + size_t size; + enum mem_buf_mem_type mem_type; + void *mem_type_data; + struct sg_table *mem_sgt; + bool secure_alloc; + int gh_rm_trans_type; + gh_memparcel_handle_t hdl; + struct list_head entry; + u32 nr_acl_entries; + int *dst_vmids; + int *dst_perms; + u32 txn_id; +}; + +/** + * struct mem_buf_desc - Internal data structure, which contains information + * about a particular memory buffer. + * @size: The size of the memory buffer + * @acl_desc: A GH ACL descriptor that describes the VMIDs that have access to + * the memory, as well as the permissions each VMID has. + * @sgl_desc: An GH SG-List descriptor that describes the IPAs of the memory + * associated with the memory buffer that was allocated from another VM. + * @memparcel_hdl: The handle associated with the memparcel that represents the + * memory buffer. + * @gh_rm_trans_type: The type of memory transfer associated with the memory (donation, + * share, lend). + * @src_mem_type: The type of memory that was allocated on the remote VM + * @src_data: Memory type specific data used by the remote VM when performing + * the allocation. + * @dst_mem_type: The memory type of the memory buffer on the native VM + * @dst_data: Memory type specific data used by the native VM when adding the + * memory to the system. + * @filp: Pointer to the file structure for the membuf + * @entry: List head for maintaing a list of memory buffers that have been + * provided by remote VMs. + * @txn: The transaction associated with a memory buffer + */ +struct mem_buf_desc { + size_t size; + struct gh_acl_desc *acl_desc; + struct gh_sgl_desc *sgl_desc; + gh_memparcel_handle_t memparcel_hdl; + int gh_rm_trans_type; + enum mem_buf_mem_type src_mem_type; + void *src_data; + enum mem_buf_mem_type dst_mem_type; + void *dst_data; + struct file *filp; + struct list_head entry; + void *txn; +}; + +struct mem_buf_xfer_dmaheap_mem { + char name[MEM_BUF_MAX_DMAHEAP_NAME_LEN]; + struct dma_buf *dmabuf; + struct dma_buf_attachment *attachment; +}; + +/* Functions invoked when treating allocation requests from other VMs. */ +static int mem_buf_rmt_alloc_dmaheap_mem(struct mem_buf_xfer_mem *xfer_mem) +{ + struct dma_buf *dmabuf; + struct dma_buf_attachment *attachment; + struct sg_table *mem_sgt; + struct mem_buf_xfer_dmaheap_mem *dmaheap_mem_data = xfer_mem->mem_type_data; + int flags = O_RDWR | O_CLOEXEC; + struct dma_heap *heap; + char *name = dmaheap_mem_data->name; + + pr_debug("%s: Starting DMAHEAP allocation\n", __func__); + heap = dma_heap_find(name); + if (!heap) { + pr_err("%s no such heap %s\n", __func__, name); + return -EINVAL; + } + + dmabuf = dma_heap_buffer_alloc(heap, xfer_mem->size, flags, 0); + if (IS_ERR(dmabuf)) { + pr_err("%s dmaheap_alloc failure sz: 0x%x heap: %s flags: 0x%x rc: %d\n", + __func__, xfer_mem->size, name, flags, + PTR_ERR(dmabuf)); + return PTR_ERR(dmabuf); + } + + attachment = dma_buf_attach(dmabuf, mem_buf_dev); + if (IS_ERR(attachment)) { + pr_err("%s dma_buf_attach failure rc: %d\n", __func__, + PTR_ERR(attachment)); + dma_buf_put(dmabuf); + return PTR_ERR(attachment); + } + + mem_sgt = dma_buf_map_attachment(attachment, DMA_BIDIRECTIONAL); + if (IS_ERR(mem_sgt)) { + pr_err("%s dma_buf_map_attachment failure rc: %d\n", __func__, + PTR_ERR(mem_sgt)); + dma_buf_detach(dmabuf, attachment); + dma_buf_put(dmabuf); + return PTR_ERR(mem_sgt); + } + + dmaheap_mem_data->dmabuf = dmabuf; + dmaheap_mem_data->attachment = attachment; + xfer_mem->mem_sgt = mem_sgt; + xfer_mem->secure_alloc = false; + + pr_debug("%s: DMAHEAP allocation complete\n", __func__); + return 0; +} + +static int mem_buf_rmt_alloc_mem(struct mem_buf_xfer_mem *xfer_mem) +{ + int ret = -EINVAL; + + if (xfer_mem->mem_type == MEM_BUF_DMAHEAP_MEM_TYPE) + ret = mem_buf_rmt_alloc_dmaheap_mem(xfer_mem); + + return ret; +} + +static void mem_buf_rmt_free_dmaheap_mem(struct mem_buf_xfer_mem *xfer_mem) +{ + struct mem_buf_xfer_dmaheap_mem *dmaheap_mem_data = xfer_mem->mem_type_data; + struct dma_buf *dmabuf = dmaheap_mem_data->dmabuf; + struct dma_buf_attachment *attachment = dmaheap_mem_data->attachment; + struct sg_table *mem_sgt = xfer_mem->mem_sgt; + + pr_debug("%s: Freeing DMAHEAP memory\n", __func__); + dma_buf_unmap_attachment(attachment, mem_sgt, DMA_BIDIRECTIONAL); + dma_buf_detach(dmabuf, attachment); + dma_buf_put(dmaheap_mem_data->dmabuf); + pr_debug("%s: DMAHEAP memory freed\n", __func__); +} + +static void mem_buf_rmt_free_mem(struct mem_buf_xfer_mem *xfer_mem) +{ + if (xfer_mem->mem_type == MEM_BUF_DMAHEAP_MEM_TYPE) + mem_buf_rmt_free_dmaheap_mem(xfer_mem); +} + +static +struct mem_buf_xfer_dmaheap_mem *mem_buf_alloc_dmaheap_xfer_mem_type_data( + void *rmt_data) +{ + struct mem_buf_xfer_dmaheap_mem *dmaheap_mem_data; + + dmaheap_mem_data = kzalloc(sizeof(*dmaheap_mem_data), GFP_KERNEL); + if (!dmaheap_mem_data) + return ERR_PTR(-ENOMEM); + + strscpy(dmaheap_mem_data->name, (char *)rmt_data, + MEM_BUF_MAX_DMAHEAP_NAME_LEN); + pr_debug("%s: DMAHEAP source heap: %s\n", __func__, + dmaheap_mem_data->name); + return dmaheap_mem_data; +} + +static void *mem_buf_alloc_xfer_mem_type_data(enum mem_buf_mem_type type, + void *rmt_data) +{ + void *data = ERR_PTR(-EINVAL); + + if (type == MEM_BUF_DMAHEAP_MEM_TYPE) + data = mem_buf_alloc_dmaheap_xfer_mem_type_data(rmt_data); + + return data; +} + +static +void mem_buf_free_dmaheap_xfer_mem_type_data(struct mem_buf_xfer_dmaheap_mem *mem) +{ + kfree(mem); +} + +static void mem_buf_free_xfer_mem_type_data(enum mem_buf_mem_type type, + void *data) +{ + if (type == MEM_BUF_DMAHEAP_MEM_TYPE) + mem_buf_free_dmaheap_xfer_mem_type_data(data); +} + +static +struct mem_buf_xfer_mem *mem_buf_prep_xfer_mem(void *req_msg) +{ + int ret; + struct mem_buf_xfer_mem *xfer_mem; + u32 nr_acl_entries; + void *arb_payload; + enum mem_buf_mem_type mem_type; + void *mem_type_data; + + nr_acl_entries = get_alloc_req_nr_acl_entries(req_msg); + if (nr_acl_entries != 1) + return ERR_PTR(-EINVAL); + + arb_payload = get_alloc_req_arb_payload(req_msg); + if (!arb_payload) + return ERR_PTR(-EINVAL); + + mem_type = get_alloc_req_src_mem_type(req_msg); + + xfer_mem = kzalloc(sizeof(*xfer_mem), GFP_KERNEL); + if (!xfer_mem) + return ERR_PTR(-ENOMEM); + + xfer_mem->txn_id = get_alloc_req_txn_id(req_msg); + xfer_mem->size = get_alloc_req_size(req_msg); + xfer_mem->mem_type = mem_type; + xfer_mem->nr_acl_entries = nr_acl_entries; + ret = mem_buf_gh_acl_desc_to_vmid_perm_list(get_alloc_req_gh_acl_desc(req_msg), + &xfer_mem->dst_vmids, + &xfer_mem->dst_perms); + if (ret) { + pr_err("%s failed to create VMID and permissions list: %d\n", + __func__, ret); + kfree(xfer_mem); + return ERR_PTR(ret); + } + mem_type_data = mem_buf_alloc_xfer_mem_type_data(mem_type, arb_payload); + if (IS_ERR(mem_type_data)) { + pr_err("%s: failed to allocate mem type specific data: %d\n", + __func__, PTR_ERR(mem_type_data)); + kfree(xfer_mem->dst_vmids); + kfree(xfer_mem->dst_perms); + kfree(xfer_mem); + return ERR_CAST(mem_type_data); + } + xfer_mem->mem_type_data = mem_type_data; + INIT_LIST_HEAD(&xfer_mem->entry); + return xfer_mem; +} + +static void mem_buf_free_xfer_mem(struct mem_buf_xfer_mem *xfer_mem) +{ + mem_buf_free_xfer_mem_type_data(xfer_mem->mem_type, + xfer_mem->mem_type_data); + kfree(xfer_mem->dst_vmids); + kfree(xfer_mem->dst_perms); + kfree(xfer_mem); +} + +/* + * @owner_vmid: Owner of the memparcel handle which has @vmids and @perms + */ +static int __maybe_unused mem_buf_get_mem_xfer_type(int *vmids, int *perms, + unsigned int nr_acl_entries, int owner_vmid) +{ + u32 i; + + for (i = 0; i < nr_acl_entries; i++) + if (vmids[i] == owner_vmid && + perms[i] != 0) + return GH_RM_TRANS_TYPE_SHARE; + + return GH_RM_TRANS_TYPE_LEND; +} + +/* + * @owner_vmid: Owner of the memparcel handle which has @acl_desc + */ +static int mem_buf_get_mem_xfer_type_gh(struct gh_acl_desc *acl_desc, int owner_vmid) +{ + u32 i, nr_acl_entries = acl_desc->n_acl_entries; + + for (i = 0; i < nr_acl_entries; i++) + if (acl_desc->acl_entries[i].vmid == owner_vmid && + acl_desc->acl_entries[i].perms != 0) + return GH_RM_TRANS_TYPE_SHARE; + + return GH_RM_TRANS_TYPE_LEND; +} + +/* + * Check whether donate operation is supported. If not, use + * lend instead. Share is not supported for remotealloc. + */ +static int get_alloc_req_xfer_type(struct mem_buf_xfer_mem *xfer_mem) +{ + static bool initialized; + static int alloc_req_xfer_type; + struct mem_buf_lend_kernel_arg arg; + int vmids[] = {VMID_TUIVM}; + int perms[] = {PERM_READ | PERM_WRITE | PERM_EXEC}; + int ret; + + if (initialized) + return alloc_req_xfer_type; + + arg.nr_acl_entries = ARRAY_SIZE(vmids); + arg.vmids = vmids; + arg.perms = perms; + arg.flags = 0; + arg.label = 0; + + ret = mem_buf_assign_mem(GH_RM_TRANS_TYPE_DONATE, xfer_mem->mem_sgt, &arg); + if (ret) { + initialized = true; + alloc_req_xfer_type = GH_RM_TRANS_TYPE_LEND; + } else { + initialized = true; + alloc_req_xfer_type = GH_RM_TRANS_TYPE_DONATE; + + mem_buf_unassign_mem(xfer_mem->mem_sgt, vmids, ARRAY_SIZE(vmids), + arg.memparcel_hdl); + } + pr_info("%s: xfer_type set to %d\n", __func__, alloc_req_xfer_type); + return alloc_req_xfer_type; +} + +static struct mem_buf_xfer_mem *mem_buf_process_alloc_req(void *req) +{ + int ret, xfer_type; + struct mem_buf_xfer_mem *xfer_mem; + struct mem_buf_lend_kernel_arg arg = {0}; + + xfer_mem = mem_buf_prep_xfer_mem(req); + if (IS_ERR(xfer_mem)) + return xfer_mem; + + ret = mem_buf_rmt_alloc_mem(xfer_mem); + if (ret < 0) + goto err_rmt_alloc; + + if (!xfer_mem->secure_alloc) { + xfer_type = get_alloc_req_xfer_type(xfer_mem); + + arg.nr_acl_entries = xfer_mem->nr_acl_entries; + arg.vmids = xfer_mem->dst_vmids; + arg.perms = xfer_mem->dst_perms; + ret = mem_buf_assign_mem(xfer_type, xfer_mem->mem_sgt, &arg); + if (ret < 0) + goto err_assign_mem; + + xfer_mem->hdl = arg.memparcel_hdl; + xfer_mem->gh_rm_trans_type = xfer_type; + } + + mutex_lock(&mem_buf_xfer_mem_list_lock); + list_add(&xfer_mem->entry, &mem_buf_xfer_mem_list); + mutex_unlock(&mem_buf_xfer_mem_list_lock); + + return xfer_mem; + +err_assign_mem: + if (ret != -EADDRNOTAVAIL) + mem_buf_rmt_free_mem(xfer_mem); +err_rmt_alloc: + mem_buf_free_xfer_mem(xfer_mem); + return ERR_PTR(ret); +} + +static void mem_buf_cleanup_alloc_req(struct mem_buf_xfer_mem *xfer_mem, + gh_memparcel_handle_t memparcel_hdl) +{ + int ret; + + if (!xfer_mem->secure_alloc) { + if (memparcel_hdl == xfer_mem->hdl) { + ret = mem_buf_unassign_mem(xfer_mem->mem_sgt, + xfer_mem->dst_vmids, + xfer_mem->nr_acl_entries, + xfer_mem->hdl); + if (ret < 0) + return; + } else { + struct gh_sgl_desc *sgl_desc; + struct gh_acl_desc *acl_desc; + size_t size; + + size = struct_size(acl_desc, acl_entries, 1); + acl_desc = kzalloc(size, GFP_KERNEL); + if (!acl_desc) + return; + + acl_desc->n_acl_entries = 1; + acl_desc->acl_entries[0].vmid = VMID_HLOS; + acl_desc->acl_entries[0].perms = GH_RM_ACL_X | GH_RM_ACL_W | GH_RM_ACL_R; + + sgl_desc = mem_buf_map_mem_s2(GH_RM_TRANS_TYPE_DONATE, &memparcel_hdl, + acl_desc, VMID_TUIVM); + if (IS_ERR(sgl_desc)) { + kfree(acl_desc); + return; + } + kvfree(sgl_desc); + kfree(acl_desc); + } + } + mem_buf_rmt_free_mem(xfer_mem); + mem_buf_free_xfer_mem(xfer_mem); +} + +static void mem_buf_alloc_req_work(struct work_struct *work) +{ + struct mem_buf_rmt_msg *rmt_msg = to_rmt_msg(work); + void *req_msg = rmt_msg->msg; + void *resp_msg; + struct mem_buf_xfer_mem *xfer_mem; + gh_memparcel_handle_t hdl = 0; + int trans_type = 0; + int ret; + + trace_receive_alloc_req(req_msg); + xfer_mem = mem_buf_process_alloc_req(req_msg); + if (IS_ERR(xfer_mem)) { + ret = PTR_ERR(xfer_mem); + pr_err("%s: failed to process rmt memory alloc request: %d\n", + __func__, ret); + xfer_mem = NULL; + } else { + ret = 0; + hdl = xfer_mem->hdl; + trans_type = xfer_mem->gh_rm_trans_type; + } + + resp_msg = mem_buf_construct_alloc_resp(req_msg, ret, hdl, trans_type); + kfree(rmt_msg->msg); + kfree(rmt_msg); + if (IS_ERR(resp_msg)) + goto out_err; + + trace_send_alloc_resp_msg(resp_msg); + ret = mem_buf_msgq_send(mem_buf_msgq_hdl, resp_msg); + /* + * Free the buffer regardless of the return value as the hypervisor + * would have consumed the data in the case of a success. + */ + kfree(resp_msg); + if (ret < 0) { + pr_err("%s: failed to send memory allocation response rc: %d\n", + __func__, ret); + goto out_err; + } + pr_debug("%s: Allocation response sent\n", __func__); + return; + +out_err: + if (xfer_mem) { + mutex_lock(&mem_buf_xfer_mem_list_lock); + list_del(&xfer_mem->entry); + mutex_unlock(&mem_buf_xfer_mem_list_lock); + mem_buf_cleanup_alloc_req(xfer_mem, xfer_mem->hdl); + } +} + +static void mem_buf_relinquish_work(struct work_struct *work) +{ + struct mem_buf_xfer_mem *xfer_mem_iter, *tmp, *xfer_mem = NULL; + struct mem_buf_rmt_msg *rmt_msg = to_rmt_msg(work); + struct mem_buf_alloc_relinquish *relinquish_msg = rmt_msg->msg; + u32 relinquish_txn_id = get_relinquish_req_txn_id(relinquish_msg); + + trace_receive_relinquish_msg(relinquish_msg); + mutex_lock(&mem_buf_xfer_mem_list_lock); + list_for_each_entry_safe(xfer_mem_iter, tmp, &mem_buf_xfer_mem_list, + entry) + if (xfer_mem_iter->txn_id == relinquish_txn_id) { + xfer_mem = xfer_mem_iter; + list_del(&xfer_mem->entry); + break; + } + mutex_unlock(&mem_buf_xfer_mem_list_lock); + + if (xfer_mem) + mem_buf_cleanup_alloc_req(xfer_mem, relinquish_msg->hdl); + else + pr_err("%s: transferred memory with txn_id 0x%x not found\n", + __func__, relinquish_txn_id); + + kfree(rmt_msg->msg); + kfree(rmt_msg); +} + +static int mem_buf_alloc_resp_hdlr(void *hdlr_data, void *msg_buf, size_t size, void *out_buf) +{ + struct mem_buf_alloc_resp *alloc_resp = msg_buf; + struct mem_buf_desc *membuf = out_buf; + int ret; + + trace_receive_alloc_resp_msg(alloc_resp); + if (!(mem_buf_capability & MEM_BUF_CAP_CONSUMER)) { + kfree(msg_buf); + return -EPERM; + } + + ret = get_alloc_resp_retval(alloc_resp); + if (ret < 0) { + pr_err("%s remote allocation failed rc: %d\n", __func__, ret); + } else { + membuf->memparcel_hdl = get_alloc_resp_hdl(alloc_resp); + membuf->gh_rm_trans_type = get_alloc_resp_trans_type(alloc_resp); + } + + kfree(msg_buf); + return ret; +} + +/* Functions invoked when treating allocation requests to other VMs. */ +static void mem_buf_alloc_req_hdlr(void *hdlr_data, void *buf, size_t size) +{ + struct mem_buf_rmt_msg *rmt_msg; + + if (!(mem_buf_capability & MEM_BUF_CAP_SUPPLIER)) { + kfree(buf); + return; + } + + rmt_msg = kmalloc(sizeof(*rmt_msg), GFP_KERNEL); + if (!rmt_msg) { + kfree(buf); + return; + } + + rmt_msg->msg = buf; + rmt_msg->msg_size = size; + INIT_WORK(&rmt_msg->work, mem_buf_alloc_req_work); + queue_work(mem_buf_wq, &rmt_msg->work); +} + +static void mem_buf_relinquish_hdlr(void *hdlr_data, void *buf, size_t size) +{ + struct mem_buf_rmt_msg *rmt_msg; + + if (!(mem_buf_capability & MEM_BUF_CAP_SUPPLIER)) { + kfree(buf); + return; + } + + rmt_msg = kmalloc(sizeof(*rmt_msg), GFP_KERNEL); + if (!rmt_msg) { + kfree(buf); + return; + } + + rmt_msg->msg = buf; + rmt_msg->msg_size = size; + INIT_WORK(&rmt_msg->work, mem_buf_relinquish_work); + queue_work(mem_buf_wq, &rmt_msg->work); +} + +static int mem_buf_request_mem(struct mem_buf_desc *membuf) +{ + void *alloc_req_msg; + int ret; + + alloc_req_msg = mem_buf_construct_alloc_req(membuf->txn, membuf->size, membuf->acl_desc, + membuf->src_mem_type, membuf->src_data); + if (IS_ERR(alloc_req_msg)) { + ret = PTR_ERR(alloc_req_msg); + goto out; + } + + ret = mem_buf_msgq_send(mem_buf_msgq_hdl, alloc_req_msg); + + /* + * Free the buffer regardless of the return value as the hypervisor + * would have consumed the data in the case of a success. + */ + kfree(alloc_req_msg); + + if (ret < 0) + goto out; + + ret = mem_buf_txn_wait(membuf->txn); + if (ret < 0) + goto out; + +out: + return ret; +} + +static void __mem_buf_relinquish_mem(u32 txn_id, u32 memparcel_hdl) +{ + void *relinquish_msg; + int ret; + + relinquish_msg = mem_buf_construct_relinquish_msg(txn_id, memparcel_hdl); + if (IS_ERR(relinquish_msg)) + return; + + trace_send_relinquish_msg(relinquish_msg); + ret = mem_buf_msgq_send(mem_buf_msgq_hdl, relinquish_msg); + + /* + * Free the buffer regardless of the return value as the hypervisor + * would have consumed the data in the case of a success. + */ + kfree(relinquish_msg); + + if (ret < 0) + pr_err("%s failed to send memory relinquish message rc: %d\n", + __func__, ret); + else + pr_debug("%s: allocation relinquish message sent\n", __func__); +} + +/* + * Check if membuf already has a valid handle. If it doesn't, then create one. + */ +static void mem_buf_relinquish_mem(struct mem_buf_desc *membuf) +{ + int ret; + int vmids[] = {VMID_HLOS}; + int perms[] = {PERM_READ | PERM_WRITE | PERM_EXEC}; + struct sg_table *sgt; + struct mem_buf_lend_kernel_arg arg; + u32 txn_id = mem_buf_retrieve_txn_id(membuf->txn); + + if (membuf->memparcel_hdl != MEM_BUF_MEMPARCEL_INVALID) { + if (membuf->gh_rm_trans_type != GH_RM_TRANS_TYPE_DONATE) { + ret = mem_buf_unmap_mem_s2(membuf->memparcel_hdl); + if (ret) + return; + } + + return __mem_buf_relinquish_mem(txn_id, membuf->memparcel_hdl); + } + + sgt = dup_gh_sgl_desc_to_sgt(membuf->sgl_desc); + if (IS_ERR(sgt)) + return; + + arg.nr_acl_entries = 1; + arg.vmids = vmids; + arg.perms = perms; + arg.flags = GH_RM_MEM_DONATE_SANITIZE; + arg.label = 0; + + ret = mem_buf_assign_mem(GH_RM_TRANS_TYPE_DONATE, sgt, &arg); + if (ret) + goto err_free_sgt; + + membuf->memparcel_hdl = arg.memparcel_hdl; + __mem_buf_relinquish_mem(txn_id, membuf->memparcel_hdl); +err_free_sgt: + sg_free_table(sgt); + kfree(sgt); +} + +static void mem_buf_relinquish_memparcel_hdl(void *hdlr_data, u32 txn_id, gh_memparcel_handle_t hdl) +{ + __mem_buf_relinquish_mem(txn_id, hdl); +} + +static int get_mem_buf(void *membuf_desc); +static int mem_buf_add_dmaheap_mem(struct mem_buf_desc *membuf, struct sg_table *sgt, + void *dst_data) +{ + char *heap_name = dst_data; + + return carveout_heap_add_memory(heap_name, sgt, (void *)membuf, + get_mem_buf, mem_buf_put); +} + +static int mem_buf_add_mem_type(struct mem_buf_desc *membuf, enum mem_buf_mem_type type, + void *dst_data, struct sg_table *sgt) +{ + if (type == MEM_BUF_DMAHEAP_MEM_TYPE) + return mem_buf_add_dmaheap_mem(membuf, sgt, dst_data); + + return -EINVAL; +} + +static int mem_buf_add_mem(struct mem_buf_desc *membuf) +{ + int i, ret, nr_sgl_entries; + struct sg_table sgt; + struct scatterlist *sgl; + u64 base, size; + + pr_debug("%s: adding memory to destination\n", __func__); + nr_sgl_entries = membuf->sgl_desc->n_sgl_entries; + ret = sg_alloc_table(&sgt, nr_sgl_entries, GFP_KERNEL); + if (ret) + return ret; + + for_each_sg(sgt.sgl, sgl, nr_sgl_entries, i) { + base = membuf->sgl_desc->sgl_entries[i].ipa_base; + size = membuf->sgl_desc->sgl_entries[i].size; + sg_set_page(sgl, phys_to_page(base), size, 0); + } + + ret = mem_buf_add_mem_type(membuf, membuf->dst_mem_type, membuf->dst_data, + &sgt); + if (ret) + pr_err("%s failed to add memory to destination rc: %d\n", + __func__, ret); + else + pr_debug("%s: memory added to destination\n", __func__); + + sg_free_table(&sgt); + return ret; +} + +static int mem_buf_remove_dmaheap_mem(struct sg_table *sgt, void *dst_data) +{ + char *heap_name = dst_data; + + return carveout_heap_remove_memory(heap_name, sgt); +} + +static int mem_buf_remove_mem_type(enum mem_buf_mem_type type, void *dst_data, + struct sg_table *sgt) +{ + if (type == MEM_BUF_DMAHEAP_MEM_TYPE) + return mem_buf_remove_dmaheap_mem(sgt, dst_data); + + return -EINVAL; +} + +static int mem_buf_remove_mem(struct mem_buf_desc *membuf) +{ + int i, ret, nr_sgl_entries; + struct sg_table sgt; + struct scatterlist *sgl; + u64 base, size; + + pr_debug("%s: removing memory from destination\n", __func__); + nr_sgl_entries = membuf->sgl_desc->n_sgl_entries; + ret = sg_alloc_table(&sgt, nr_sgl_entries, GFP_KERNEL); + if (ret) + return ret; + + for_each_sg(sgt.sgl, sgl, nr_sgl_entries, i) { + base = membuf->sgl_desc->sgl_entries[i].ipa_base; + size = membuf->sgl_desc->sgl_entries[i].size; + sg_set_page(sgl, phys_to_page(base), size, 0); + } + + ret = mem_buf_remove_mem_type(membuf->dst_mem_type, membuf->dst_data, + &sgt); + if (ret) + pr_err("%s failed to remove memory from destination rc: %d\n", + __func__, ret); + else + pr_debug("%s: memory removed from destination\n", __func__); + + sg_free_table(&sgt); + return ret; +} + +static void *mem_buf_retrieve_dmaheap_mem_type_data_user( + struct mem_buf_dmaheap_data __user *udata) +{ + char *buf; + int ret; + struct mem_buf_dmaheap_data data; + + ret = copy_struct_from_user(&data, sizeof(data), + udata, + sizeof(data)); + if (ret) + return ERR_PTR(-EINVAL); + + buf = kcalloc(MEM_BUF_MAX_DMAHEAP_NAME_LEN, sizeof(*buf), GFP_KERNEL); + if (!buf) + return ERR_PTR(-ENOMEM); + + ret = strncpy_from_user(buf, (const void __user *)data.heap_name, + MEM_BUF_MAX_DMAHEAP_NAME_LEN); + if (ret < 0 || ret == MEM_BUF_MAX_DMAHEAP_NAME_LEN) { + kfree(buf); + return ERR_PTR(-EINVAL); + } + return buf; +} + +static void *mem_buf_retrieve_mem_type_data_user(enum mem_buf_mem_type mem_type, + void __user *mem_type_data) +{ + void *data = ERR_PTR(-EINVAL); + + if (mem_type == MEM_BUF_DMAHEAP_MEM_TYPE) + data = mem_buf_retrieve_dmaheap_mem_type_data_user(mem_type_data); + + return data; +} + +static void *mem_buf_retrieve_dmaheap_mem_type_data(char *dmaheap_name) +{ + return kstrdup(dmaheap_name, GFP_KERNEL); +} + +static void *mem_buf_retrieve_mem_type_data(enum mem_buf_mem_type mem_type, + void *mem_type_data) +{ + void *data = ERR_PTR(-EINVAL); + + if (mem_type == MEM_BUF_DMAHEAP_MEM_TYPE) + data = mem_buf_retrieve_dmaheap_mem_type_data(mem_type_data); + + return data; +} + +static void mem_buf_free_dmaheap_mem_type_data(char *dmaheap_name) +{ + kfree(dmaheap_name); +} + +static void mem_buf_free_mem_type_data(enum mem_buf_mem_type mem_type, + void *mem_type_data) +{ + if (mem_type == MEM_BUF_DMAHEAP_MEM_TYPE) + mem_buf_free_dmaheap_mem_type_data(mem_type_data); +} + +static int mem_buf_buffer_release(struct inode *inode, struct file *filp) +{ + struct mem_buf_desc *membuf = filp->private_data; + int ret; + + mutex_lock(&mem_buf_list_lock); + list_del(&membuf->entry); + mutex_unlock(&mem_buf_list_lock); + + pr_debug("%s: Destroyng carveout memory\n", __func__); + ret = mem_buf_remove_mem(membuf); + if (ret < 0) + goto out_free_mem; + + ret = mem_buf_unmap_mem_s1(membuf->sgl_desc); + if (ret < 0) + goto out_free_mem; + + mem_buf_relinquish_mem(membuf); + +out_free_mem: + mem_buf_destroy_txn(mem_buf_msgq_hdl, membuf->txn); + mem_buf_free_mem_type_data(membuf->dst_mem_type, membuf->dst_data); + mem_buf_free_mem_type_data(membuf->src_mem_type, membuf->src_data); + kvfree(membuf->sgl_desc); + kfree(membuf->acl_desc); + kfree(membuf); + return ret; +} + +static const struct file_operations mem_buf_fops = { + .release = mem_buf_buffer_release, +}; + +static bool is_valid_mem_type(enum mem_buf_mem_type mem_type) +{ + return mem_type == MEM_BUF_DMAHEAP_MEM_TYPE; +} + +static void *mem_buf_alloc(struct mem_buf_allocation_data *alloc_data) +{ + int ret; + struct file *filp; + struct mem_buf_desc *membuf; + struct gh_sgl_desc *sgl_desc; + int op; + int perms = PERM_READ | PERM_WRITE | PERM_EXEC; + + if (!(mem_buf_capability & MEM_BUF_CAP_CONSUMER)) + return ERR_PTR(-EOPNOTSUPP); + + if (!alloc_data || !alloc_data->size || alloc_data->nr_acl_entries != 1 || + !alloc_data->vmids || !alloc_data->perms || + !is_valid_mem_type(alloc_data->src_mem_type) || + !is_valid_mem_type(alloc_data->dst_mem_type)) + return ERR_PTR(-EINVAL); + + membuf = kzalloc(sizeof(*membuf), GFP_KERNEL); + if (!membuf) + return ERR_PTR(-ENOMEM); + + pr_debug("%s: mem buf alloc begin\n", __func__); + membuf->size = ALIGN(alloc_data->size, MEM_BUF_MHP_ALIGNMENT); + membuf->acl_desc = mem_buf_vmid_perm_list_to_gh_acl( + alloc_data->vmids, &perms, + alloc_data->nr_acl_entries); + if (IS_ERR(membuf->acl_desc)) { + ret = PTR_ERR(membuf->acl_desc); + goto err_alloc_acl_list; + } + membuf->src_mem_type = alloc_data->src_mem_type; + membuf->dst_mem_type = alloc_data->dst_mem_type; + + membuf->src_data = + mem_buf_retrieve_mem_type_data(alloc_data->src_mem_type, + alloc_data->src_data); + if (IS_ERR(membuf->src_data)) { + ret = PTR_ERR(membuf->src_data); + goto err_alloc_src_data; + } + + membuf->dst_data = + mem_buf_retrieve_mem_type_data(alloc_data->dst_mem_type, + alloc_data->dst_data); + if (IS_ERR(membuf->dst_data)) { + ret = PTR_ERR(membuf->dst_data); + goto err_alloc_dst_data; + } + + membuf->txn = mem_buf_init_txn(mem_buf_msgq_hdl, membuf); + if (IS_ERR(membuf->txn)) { + ret = PTR_ERR(membuf->txn); + goto err_init_txn; + } + + trace_mem_buf_alloc_info(membuf->size, membuf->src_mem_type, + membuf->dst_mem_type, membuf->acl_desc); + ret = mem_buf_request_mem(membuf); + if (ret) + goto err_mem_req; + + op = membuf->gh_rm_trans_type; + sgl_desc = mem_buf_map_mem_s2(op, &membuf->memparcel_hdl, membuf->acl_desc, VMID_HLOS); + if (IS_ERR(sgl_desc)) + goto err_map_mem_s2; + membuf->sgl_desc = sgl_desc; + + ret = mem_buf_map_mem_s1(membuf->sgl_desc); + if (ret) + goto err_map_mem_s1; + + filp = anon_inode_getfile("membuf", &mem_buf_fops, membuf, O_RDWR); + if (IS_ERR(filp)) { + ret = PTR_ERR(filp); + goto err_get_file; + } + membuf->filp = filp; + + mutex_lock(&mem_buf_list_lock); + list_add_tail(&membuf->entry, &mem_buf_list); + mutex_unlock(&mem_buf_list_lock); + + ret = mem_buf_add_mem(membuf); + if (ret) + goto err_add_mem; + + pr_debug("%s: mem buf alloc success\n", __func__); + return membuf; + +err_add_mem: + fput(filp); + return ERR_PTR(ret); +err_get_file: + if (mem_buf_unmap_mem_s1(membuf->sgl_desc) < 0) { + kvfree(membuf->sgl_desc); + goto err_mem_req; + } +err_map_mem_s1: +err_map_mem_s2: + mem_buf_relinquish_mem(membuf); + kvfree(membuf->sgl_desc); +err_mem_req: + mem_buf_destroy_txn(mem_buf_msgq_hdl, membuf->txn); +err_init_txn: + mem_buf_free_mem_type_data(membuf->dst_mem_type, membuf->dst_data); +err_alloc_dst_data: + mem_buf_free_mem_type_data(membuf->src_mem_type, membuf->src_data); +err_alloc_src_data: + kfree(membuf->acl_desc); +err_alloc_acl_list: + kfree(membuf); + return ERR_PTR(ret); +} + +static int _mem_buf_get_fd(struct file *filp) +{ + int fd; + + if (!filp) + return -EINVAL; + + fd = get_unused_fd_flags(O_CLOEXEC); + if (fd < 0) + return fd; + + fd_install(fd, filp); + return fd; +} + +int mem_buf_get_fd(void *membuf_desc) +{ + struct mem_buf_desc *membuf = membuf_desc; + + if (!membuf_desc) + return -EINVAL; + + return _mem_buf_get_fd(membuf->filp); +} +EXPORT_SYMBOL(mem_buf_get_fd); + +static void _mem_buf_put(struct file *filp) +{ + fput(filp); +} + +void mem_buf_put(void *membuf_desc) +{ + struct mem_buf_desc *membuf = membuf_desc; + + if (membuf && membuf->filp) + _mem_buf_put(membuf->filp); +} +EXPORT_SYMBOL(mem_buf_put); + +static bool is_mem_buf_file(struct file *filp) +{ + return filp->f_op == &mem_buf_fops; +} + +void *mem_buf_get(int fd) +{ + struct file *filp; + + filp = fget(fd); + + if (!filp) + return ERR_PTR(-EBADF); + + if (!is_mem_buf_file(filp)) { + fput(filp); + return ERR_PTR(-EINVAL); + } + + return filp->private_data; +} +EXPORT_SYMBOL(mem_buf_get); + +static int get_mem_buf(void *membuf_desc) +{ + struct mem_buf_desc *membuf = membuf_desc; + + /* + * get_file_rcu used for atomic_long_inc_unless_zero. + * It returns 0 if file count is already zero. + */ + if (!get_file_rcu(membuf->filp)) + return -EINVAL; + + return 0; +} + +static void mem_buf_retrieve_release(struct qcom_sg_buffer *buffer) +{ + sg_free_table(&buffer->sg_table); + kfree(buffer); +} + +struct dma_buf *mem_buf_retrieve(struct mem_buf_retrieve_kernel_arg *arg) +{ + int ret, op; + struct qcom_sg_buffer *buffer; + struct gh_acl_desc *acl_desc; + struct gh_sgl_desc *sgl_desc; + DEFINE_DMA_BUF_EXPORT_INFO(exp_info); + struct dma_buf *dmabuf; + struct sg_table *sgt; + + if (arg->fd_flags & ~MEM_BUF_VALID_FD_FLAGS) + return ERR_PTR(-EINVAL); + + if (!arg->nr_acl_entries || !arg->vmids || !arg->perms) + return ERR_PTR(-EINVAL); + + buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); + if (!buffer) + return ERR_PTR(-ENOMEM); + + acl_desc = mem_buf_vmid_perm_list_to_gh_acl(arg->vmids, arg->perms, + arg->nr_acl_entries); + if (IS_ERR(acl_desc)) { + ret = PTR_ERR(acl_desc); + goto err_gh_acl; + } + + op = mem_buf_get_mem_xfer_type_gh(acl_desc, arg->sender_vmid); + sgl_desc = mem_buf_map_mem_s2(op, &arg->memparcel_hdl, acl_desc, arg->sender_vmid); + if (IS_ERR(sgl_desc)) { + ret = PTR_ERR(sgl_desc); + goto err_map_s2; + } + + ret = mem_buf_map_mem_s1(sgl_desc); + if (ret < 0) + goto err_map_mem_s1; + + sgt = dup_gh_sgl_desc_to_sgt(sgl_desc); + if (IS_ERR(sgt)) { + ret = PTR_ERR(sgt); + goto err_dup_sgt; + } + buffer->sg_table = *sgt; + kfree(sgt); + + INIT_LIST_HEAD(&buffer->attachments); + mutex_init(&buffer->lock); + buffer->heap = NULL; + buffer->len = mem_buf_get_sgl_buf_size(sgl_desc); + buffer->uncached = false; + buffer->free = mem_buf_retrieve_release; + buffer->vmperm = mem_buf_vmperm_alloc_accept(&buffer->sg_table, + arg->memparcel_hdl); + + exp_info.size = buffer->len; + exp_info.flags = arg->fd_flags; + exp_info.priv = buffer; + + dmabuf = mem_buf_dma_buf_export(&exp_info, &qcom_sg_buf_ops); + if (IS_ERR(dmabuf)) { + ret = PTR_ERR(dmabuf); + goto err_export_dma_buf; + } + + /* sgt & qcom_sg_buffer will be freed by mem_buf_retrieve_release */ + kvfree(sgl_desc); + kfree(acl_desc); + return dmabuf; + +err_export_dma_buf: + sg_free_table(&buffer->sg_table); +err_dup_sgt: + mem_buf_unmap_mem_s1(sgl_desc); +err_map_mem_s1: + kvfree(sgl_desc); + mem_buf_unmap_mem_s2(arg->memparcel_hdl); +err_map_s2: + kfree(acl_desc); +err_gh_acl: + kfree(buffer); + return ERR_PTR(ret); +} +EXPORT_SYMBOL(mem_buf_retrieve); + +static int mem_buf_prep_alloc_data(struct mem_buf_allocation_data *alloc_data, + struct mem_buf_alloc_ioctl_arg *allocation_args) +{ + unsigned int nr_acl_entries = allocation_args->nr_acl_entries; + int ret; + + alloc_data->size = allocation_args->size; + alloc_data->nr_acl_entries = nr_acl_entries; + + ret = mem_buf_acl_to_vmid_perms_list(nr_acl_entries, + (const void __user *)allocation_args->acl_list, + &alloc_data->vmids, &alloc_data->perms); + if (ret) + goto err_acl; + + alloc_data->src_mem_type = allocation_args->src_mem_type; + alloc_data->dst_mem_type = allocation_args->dst_mem_type; + + alloc_data->src_data = + mem_buf_retrieve_mem_type_data_user( + allocation_args->src_mem_type, + (void __user *)allocation_args->src_data); + if (IS_ERR(alloc_data->src_data)) { + ret = PTR_ERR(alloc_data->src_data); + goto err_alloc_src_data; + } + + alloc_data->dst_data = + mem_buf_retrieve_mem_type_data_user( + allocation_args->dst_mem_type, + (void __user *)allocation_args->dst_data); + if (IS_ERR(alloc_data->dst_data)) { + ret = PTR_ERR(alloc_data->dst_data); + goto err_alloc_dst_data; + } + + return 0; + +err_alloc_dst_data: + mem_buf_free_mem_type_data(alloc_data->src_mem_type, + alloc_data->src_data); +err_alloc_src_data: + kfree(alloc_data->vmids); + kfree(alloc_data->perms); +err_acl: + return ret; +} + +static void mem_buf_free_alloc_data(struct mem_buf_allocation_data *alloc_data) +{ + mem_buf_free_mem_type_data(alloc_data->dst_mem_type, + alloc_data->dst_data); + mem_buf_free_mem_type_data(alloc_data->src_mem_type, + alloc_data->src_data); + kfree(alloc_data->vmids); + kfree(alloc_data->perms); +} + +int mem_buf_alloc_fd(struct mem_buf_alloc_ioctl_arg *allocation_args) +{ + struct mem_buf_desc *membuf; + struct mem_buf_allocation_data alloc_data; + int ret; + + if (!allocation_args->size || !allocation_args->nr_acl_entries || + !allocation_args->acl_list || + (allocation_args->nr_acl_entries > MEM_BUF_MAX_NR_ACL_ENTS) || + !is_valid_mem_type(allocation_args->src_mem_type) || + !is_valid_mem_type(allocation_args->dst_mem_type) || + allocation_args->reserved0 || allocation_args->reserved1 || + allocation_args->reserved2) + return -EINVAL; + + ret = mem_buf_prep_alloc_data(&alloc_data, allocation_args); + if (ret < 0) + return ret; + + membuf = mem_buf_alloc(&alloc_data); + if (IS_ERR(membuf)) { + ret = PTR_ERR(membuf); + goto out; + } + + ret = mem_buf_get_fd(membuf); + if (ret < 0) + mem_buf_put(membuf); + +out: + mem_buf_free_alloc_data(&alloc_data); + return ret; +} + +int mem_buf_retrieve_user(struct mem_buf_retrieve_ioctl_arg *uarg) +{ + int ret, fd; + int *vmids, *perms; + struct dma_buf *dmabuf; + struct mem_buf_retrieve_kernel_arg karg = {0}; + + if (!uarg->nr_acl_entries || !uarg->acl_list || + uarg->nr_acl_entries > MEM_BUF_MAX_NR_ACL_ENTS || + uarg->reserved0 || uarg->reserved1 || + uarg->reserved2 || + uarg->fd_flags & ~MEM_BUF_VALID_FD_FLAGS) + return -EINVAL; + + ret = mem_buf_acl_to_vmid_perms_list(uarg->nr_acl_entries, + (void *)uarg->acl_list, &vmids, &perms); + if (ret) + return ret; + + karg.sender_vmid = mem_buf_fd_to_vmid(uarg->sender_vm_fd); + if (karg.sender_vmid < 0) { + pr_err_ratelimited("%s: Invalid sender_vmid %d\n", __func__, uarg->sender_vm_fd); + goto err_sender_vmid; + } + + karg.nr_acl_entries = uarg->nr_acl_entries; + karg.vmids = vmids; + karg.perms = perms; + karg.memparcel_hdl = uarg->memparcel_hdl; + karg.fd_flags = uarg->fd_flags; + dmabuf = mem_buf_retrieve(&karg); + if (IS_ERR(dmabuf)) { + ret = PTR_ERR(dmabuf); + goto err_retrieve; + } + + fd = dma_buf_fd(dmabuf, karg.fd_flags); + if (fd < 0) { + ret = fd; + goto err_fd; + } + + uarg->dma_buf_import_fd = fd; + kfree(vmids); + kfree(perms); + return 0; +err_fd: + dma_buf_put(dmabuf); +err_sender_vmid: +err_retrieve: + kfree(vmids); + kfree(perms); + return ret; +} + +static const struct mem_buf_msgq_ops msgq_ops = { + .alloc_req_hdlr = mem_buf_alloc_req_hdlr, + .alloc_resp_hdlr = mem_buf_alloc_resp_hdlr, + .relinquish_hdlr = mem_buf_relinquish_hdlr, + .relinquish_memparcel_hdl = mem_buf_relinquish_memparcel_hdl, +}; + +int mem_buf_msgq_alloc(struct device *dev) +{ + struct mem_buf_msgq_hdlr_info info = { + .msgq_ops = &msgq_ops, + }; + int ret; + + /* No msgq if neither a consumer nor a supplier */ + if (!(mem_buf_capability & MEM_BUF_CAP_DUAL)) + return 0; + + mem_buf_wq = alloc_workqueue("mem_buf_wq", WQ_HIGHPRI | WQ_UNBOUND, 0); + if (!mem_buf_wq) { + dev_err(dev, "Unable to initialize workqueue\n"); + return -EINVAL; + } + + mem_buf_msgq_hdl = mem_buf_msgq_register("trusted_vm", &info); + if (IS_ERR(mem_buf_msgq_hdl)) { + ret = PTR_ERR(mem_buf_msgq_hdl); + dev_err(dev, "Unable to register for mem-buf message queue\n"); + goto err_msgq_register; + } + + return 0; + +err_msgq_register: + destroy_workqueue(mem_buf_wq); + mem_buf_wq = NULL; + return ret; +} + +void mem_buf_msgq_free(struct device *dev) +{ + if (!(mem_buf_capability & MEM_BUF_CAP_DUAL)) + return; + + mutex_lock(&mem_buf_list_lock); + if (!list_empty(&mem_buf_list)) + dev_err(mem_buf_dev, + "Removing mem-buf driver while there are membufs\n"); + mutex_unlock(&mem_buf_list_lock); + + mutex_lock(&mem_buf_xfer_mem_list_lock); + if (!list_empty(&mem_buf_xfer_mem_list)) + dev_err(mem_buf_dev, + "Removing mem-buf driver while memory is still lent\n"); + mutex_unlock(&mem_buf_xfer_mem_list_lock); + mem_buf_msgq_unregister(mem_buf_msgq_hdl); + mem_buf_msgq_hdl = NULL; + destroy_workqueue(mem_buf_wq); + mem_buf_wq = NULL; +} diff --git a/drivers/soc/qcom/mem_buf/mem-buf-gh.h b/drivers/soc/qcom/mem_buf/mem-buf-gh.h new file mode 100644 index 000000000000..4d19a17b0663 --- /dev/null +++ b/drivers/soc/qcom/mem_buf/mem-buf-gh.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef MEM_BUF_GH_H +#define MEM_BUF_GH_H + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "mem-buf-dev.h" + +int mem_buf_acl_to_vmid_perms_list(unsigned int nr_acl_entries, const void __user *acl_entries, + int **dst_vmids, int **dst_perms); + +#if IS_ENABLED(CONFIG_QCOM_MEM_BUF_GH) +#include + +int mem_buf_alloc_fd(struct mem_buf_alloc_ioctl_arg *allocation_args); +int mem_buf_retrieve_user(struct mem_buf_retrieve_ioctl_arg *uarg); +int mem_buf_msgq_alloc(struct device *dev); +void mem_buf_msgq_free(struct device *dev); +#else +static inline int mem_buf_alloc_fd(struct mem_buf_alloc_ioctl_arg *allocation_args) +{ + return -EOPNOTSUPP; +} + +static inline int mem_buf_retrieve_user(struct mem_buf_retrieve_ioctl_arg *uarg) +{ + return -EOPNOTSUPP; +} + +static inline int mem_buf_msgq_alloc(struct device *dev) +{ + return 0; +} + +static inline void mem_buf_msgq_free(struct device *dev) +{ +} +#endif +#endif diff --git a/drivers/soc/qcom/mem_buf/mem-buf-ids.c b/drivers/soc/qcom/mem_buf/mem-buf-ids.c new file mode 100644 index 000000000000..a81f7f7bf347 --- /dev/null +++ b/drivers/soc/qcom/mem_buf/mem-buf-ids.c @@ -0,0 +1,340 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) "mem_buf_vm: " fmt + +#include +#include +#include +#include "mem-buf-dev.h" +#include "mem-buf-ids.h" + +#define DEVNAME "mem_buf_vm" +#define NUM_MEM_BUF_VM_MINORS 128 + +static dev_t mem_buf_vm_devt; +static struct class *mem_buf_vm_class; + +/* + * VM objects have the same lifetime as this module. + */ +static DEFINE_XARRAY_ALLOC(mem_buf_vm_minors); +static DEFINE_XARRAY(mem_buf_vms); +int current_vmid; + +#define PERIPHERAL_VM(_uname, _lname) \ +static struct mem_buf_vm vm_ ## _lname = { \ + .name = "qcom," #_lname, \ + .vmid = VMID_ ## _uname, \ + .allowed_api = 0, \ +} + +PERIPHERAL_VM(CP_TOUCH, cp_touch); +PERIPHERAL_VM(CP_BITSTREAM, cp_bitstream); +PERIPHERAL_VM(CP_PIXEL, cp_pixel); +PERIPHERAL_VM(CP_NON_PIXEL, cp_non_pixel); +PERIPHERAL_VM(CP_CAMERA, cp_camera); +PERIPHERAL_VM(CP_SEC_DISPLAY, cp_sec_display); +PERIPHERAL_VM(CP_SPSS_SP, cp_spss_sp); +PERIPHERAL_VM(CP_CAMERA_PREVIEW, cp_camera_preview); +PERIPHERAL_VM(CP_SPSS_SP_SHARED, cp_spss_sp_shared); +PERIPHERAL_VM(CP_SPSS_HLOS_SHARED, cp_spss_hlos_shared); +PERIPHERAL_VM(CP_CDSP, cp_cdsp); +PERIPHERAL_VM(CP_APP, cp_app); + +static struct mem_buf_vm vm_trusted_vm = { + .name = "qcom,trusted_vm", + .vmid = VMID_TUIVM, + .allowed_api = MEM_BUF_API_GUNYAH, +}; + +static struct mem_buf_vm vm_oemvm = { + .name = "qcom,oemvm", + .vmid = VMID_OEMVM, + .allowed_api = MEM_BUF_API_GUNYAH, +}; + +static struct mem_buf_vm vm_hlos = { + .name = "qcom,hlos", + .vmid = VMID_HLOS, + .allowed_api = 0, +}; + +struct mem_buf_vm *pdata_array[] = { + &vm_trusted_vm, + &vm_oemvm, + &vm_hlos, + &vm_cp_touch, + &vm_cp_bitstream, + &vm_cp_pixel, + &vm_cp_non_pixel, + &vm_cp_camera, + &vm_cp_sec_display, + &vm_cp_spss_sp, + &vm_cp_camera_preview, + &vm_cp_spss_sp_shared, + &vm_cp_spss_hlos_shared, + &vm_cp_cdsp, + &vm_cp_app, + NULL, +}; + +/* + * Opening this file acquires a refcount on vm->dev's kobject - see + * chrdev_open(). So private data won't be free'd out from + * under us. + */ +static int mem_buf_vm_open(struct inode *inode, struct file *file) +{ + struct mem_buf_vm *vm; + + vm = container_of(inode->i_cdev, struct mem_buf_vm, cdev); + file->private_data = vm; + return 0; +} + +static const struct file_operations mem_buf_vm_fops = { + .open = mem_buf_vm_open, +}; + +bool mem_buf_vm_uses_hyp_assign(void) +{ + return current_vmid == VMID_HLOS; +} +EXPORT_SYMBOL(mem_buf_vm_uses_hyp_assign); + +/* + * Use Gunyah API if any vm in the source or destination requires it. + */ +int mem_buf_vm_uses_gunyah(int *vmids, unsigned int nr_acl_entries) +{ + struct mem_buf_vm *vm; + int i; + + for (i = 0; i < nr_acl_entries; i++) { + vm = xa_load(&mem_buf_vms, vmids[i]); + if (!vm) { + pr_err_ratelimited("No vm with vmid=0x%x\n", vmids[i]); + return -EINVAL; + } + + if (vm->allowed_api & MEM_BUF_API_GUNYAH) + return true; + } + + vm = xa_load(&mem_buf_vms, current_vmid); + if (!vm) { + pr_err_ratelimited("No vm with vmid=0x%x\n", current_vmid); + return PTR_ERR(vm); + } + + if (vm->allowed_api & MEM_BUF_API_GUNYAH) + return true; + + return false; +} +EXPORT_SYMBOL(mem_buf_vm_uses_gunyah); + +int mem_buf_fd_to_vmid(int fd) +{ + int ret = -EINVAL; + struct mem_buf_vm *vm; + struct file *file; + + file = fget(fd); + if (!file) + return -EINVAL; + + if (file->f_op != &mem_buf_vm_fops) { + pr_err_ratelimited("Invalid vm file type\n"); + fput(file); + return -EINVAL; + } + + vm = file->private_data; + ret = vm->vmid; + fput(file); + return ret; +} +EXPORT_SYMBOL(mem_buf_fd_to_vmid); + +static void mem_buf_vm_device_release(struct device *dev) +{ + struct mem_buf_vm *vm; + + vm = container_of(dev, struct mem_buf_vm, dev); + kfree(vm); +} + +/* + * caller must fill in all fields of new_vm except for cdev & dev. + */ +static int mem_buf_vm_add(struct mem_buf_vm *new_vm) +{ + struct mem_buf_vm *vm; + struct device *dev; + int minor, ret; + unsigned long idx; + + xa_for_each(&mem_buf_vm_minors, idx, vm) { + if (!strcmp(vm->name, new_vm->name)) { + pr_err("duplicate vm %s\n", vm->name); + ret = -EINVAL; + goto err_duplicate; + } + } + + ret = xa_alloc(&mem_buf_vm_minors, &minor, new_vm, + XA_LIMIT(0, NUM_MEM_BUF_VM_MINORS - 1), GFP_KERNEL); + if (ret < 0) { + pr_err("no more minors\n"); + goto err_devt; + } + + cdev_init(&new_vm->cdev, &mem_buf_vm_fops); + dev = &new_vm->dev; + device_initialize(dev); + dev->devt = MKDEV(MAJOR(mem_buf_vm_devt), minor); + dev->class = mem_buf_vm_class; + dev->parent = NULL; + dev->release = mem_buf_vm_device_release; + dev_set_drvdata(dev, new_vm); + dev_set_name(dev, "%s", new_vm->name); + + ret = xa_err(xa_store(&mem_buf_vms, new_vm->vmid, new_vm, GFP_KERNEL)); + if (ret) + goto err_xa_store; + + ret = cdev_device_add(&new_vm->cdev, dev); + if (ret) { + pr_err("Adding cdev %s failed\n", new_vm->name); + goto err_cdev_add; + } + return 0; + +err_cdev_add: + xa_erase(&mem_buf_vms, new_vm->vmid); +err_xa_store: + xa_erase(&mem_buf_vm_minors, minor); + put_device(dev); +err_devt: +err_duplicate: + return ret; +} + +static int mem_buf_vm_add_pdata(struct mem_buf_vm *pdata) +{ + struct mem_buf_vm *vm; + int ret; + + vm = kmemdup(pdata, sizeof(*vm), GFP_KERNEL); + if (!vm) + return -ENOMEM; + + ret = mem_buf_vm_add(vm); + if (ret) { + kfree(vm); + return ret; + } + return 0; +} + +static int mem_buf_vm_add_self(void) +{ + struct mem_buf_vm *vm, *self; + int ret; + + vm = xa_load(&mem_buf_vms, current_vmid); + if (!vm) + return PTR_ERR(vm); + + self = kzalloc(sizeof(*self), GFP_KERNEL); + if (!self) + return -ENOMEM; + + /* Create an aliased name */ + self->name = "qcom,self"; + self->vmid = vm->vmid; + self->allowed_api = vm->allowed_api; + + ret = mem_buf_vm_add(self); + if (ret) { + kfree(self); + return ret; + } + return 0; +} + +static char *mem_buf_vm_devnode(struct device *dev, umode_t *mode) +{ + return kasprintf(GFP_KERNEL, "mem_buf_vm/%s", dev_name(dev)); +} + +static int mem_buf_vm_put_class_device_cb(struct device *dev, void *data) +{ + struct mem_buf_vm *vm = container_of(dev, struct mem_buf_vm, dev); + + cdev_device_del(&vm->cdev, dev); + return 0; +} + +int mem_buf_vm_init(struct device *dev) +{ + struct mem_buf_vm **p; + int ret, vmid; + + ret = of_property_read_u32(dev->of_node, "qcom,vmid", &vmid); + if (ret) { + dev_err(dev, "missing qcom,vmid property\n"); + return ret; + } + current_vmid = vmid; + + ret = alloc_chrdev_region(&mem_buf_vm_devt, 0, NUM_MEM_BUF_VM_MINORS, + DEVNAME); + if (ret) + return ret; + + mem_buf_vm_class = class_create(THIS_MODULE, DEVNAME); + if (IS_ERR(mem_buf_vm_class)) { + ret = PTR_ERR(mem_buf_vm_class); + goto err_class_create; + } + mem_buf_vm_class->devnode = mem_buf_vm_devnode; + + for (p = pdata_array; *p; p++) { + ret = mem_buf_vm_add_pdata(*p); + if (ret) + goto err_pdata; + } + + ret = mem_buf_vm_add_self(); + if (ret) + goto err_self; + + return 0; + +err_self: +err_pdata: + xa_destroy(&mem_buf_vms); + xa_destroy(&mem_buf_vm_minors); + class_for_each_device(mem_buf_vm_class, NULL, NULL, + mem_buf_vm_put_class_device_cb); + class_destroy(mem_buf_vm_class); +err_class_create: + unregister_chrdev_region(mem_buf_vm_devt, NUM_MEM_BUF_VM_MINORS); + return ret; +} + +void mem_buf_vm_exit(void) +{ + xa_destroy(&mem_buf_vms); + xa_destroy(&mem_buf_vm_minors); + class_for_each_device(mem_buf_vm_class, NULL, NULL, + mem_buf_vm_put_class_device_cb); + class_destroy(mem_buf_vm_class); + unregister_chrdev_region(mem_buf_vm_devt, NUM_MEM_BUF_VM_MINORS); +} diff --git a/drivers/soc/qcom/mem_buf/mem-buf-ids.h b/drivers/soc/qcom/mem_buf/mem-buf-ids.h new file mode 100644 index 000000000000..86c61e026341 --- /dev/null +++ b/drivers/soc/qcom/mem_buf/mem-buf-ids.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2021-2022, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef MEM_BUF_IDS_H +#define MEM_BUF_IDS_H + +#include +#include + +#define MEM_BUF_API_HYP_ASSIGN BIT(0) +#define MEM_BUF_API_GUNYAH BIT(1) + +/* Future targets should receive a notification with the proper value */ +#define VMID_TUIVM (45) +#define VMID_OEMVM (49) + +/* + * @vmid - id assigned by hypervisor to uniquely identify a VM + * @allowed_api - Some vms may use a different hypervisor interface. + */ +struct mem_buf_vm { + const char *name; + u16 vmid; + u32 allowed_api; + struct cdev cdev; + struct device dev; +}; + +extern int current_vmid; +int mem_buf_vm_init(struct device *dev); +void mem_buf_vm_exit(void); + +bool mem_buf_vm_uses_hyp_assign(void); +/* + * Returns a negative number for invalid arguments, otherwise a positive value + * if gunyah APIs are required. + */ +int mem_buf_vm_uses_gunyah(int *vmids, unsigned int nr_acl_entries); + +/* @Return: A negative number on failure, or vmid on success */ +int mem_buf_fd_to_vmid(int fd); + +#endif diff --git a/drivers/soc/qcom/mem_buf/mem-buf-msgq.c b/drivers/soc/qcom/mem_buf/mem-buf-msgq.c new file mode 100644 index 000000000000..1f257b9234ff --- /dev/null +++ b/drivers/soc/qcom/mem_buf/mem-buf-msgq.c @@ -0,0 +1,453 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mem-buf-msgq.h" +#include "trace-mem-buf.h" + +#define MEM_BUF_TIMEOUT_MS 3500 + +/* + * Data structures for tracking request/reply transactions, as well as message + * queue usage + */ +static DEFINE_MUTEX(mem_buf_msgq_list_lock); +static LIST_HEAD(mem_buf_msgq_list); + +struct mem_buf_msgq_id { + const char *name; + int label; +}; + +static struct mem_buf_msgq_id mem_buf_msgqs[] = { + { + .name = "trusted_vm", + .label = GH_MSGQ_LABEL_MEMBUF, + }, + { + }, +}; + +/** + * struct mem_buf_txn: Represents a transaction (request/response pair) in the + * mem-buf-msgq driver. + * @txn_id: Transaction ID used to match requests and responses (i.e. a new ID + * is allocated per request, and the response will have a matching ID). + * @txn_ret: The return value of the transaction. + * @txn_done: Signals that a response has arrived. + * @resp_buf: A pointer to a buffer where the response should be decoded into. + */ +struct mem_buf_txn { + int txn_id; + int txn_ret; + struct completion txn_done; + void *resp_buf; +}; + +struct mem_buf_msgq_desc { + const struct mem_buf_msgq_ops *msgq_ops; + void *hdlr_data; + struct mutex idr_mutex; + struct idr txn_idr; + void *msgq_hdl; + struct task_struct *recv_thr; + struct list_head list; +}; + +static size_t mem_buf_get_mem_type_alloc_req_size(enum mem_buf_mem_type type) +{ + if (type == MEM_BUF_DMAHEAP_MEM_TYPE) + return MEM_BUF_MAX_DMAHEAP_NAME_LEN; + + return 0; +} + +static void mem_buf_populate_alloc_req_arb_payload(void *dst, void *src, + enum mem_buf_mem_type type) +{ + if (type == MEM_BUF_DMAHEAP_MEM_TYPE) + strscpy(dst, src, MEM_BUF_MAX_DMAHEAP_NAME_LEN); +} + +/* + * mem_buf_construct_alloc_req: Constructs an allocation request message. + * @mem_buf_txn: A valid transaction structure allocated by a call to mem_buf_init_txn(). + * @alloc_size: The size of the allocation to be requested. + * @acl_desc: A GH ACL descriptor that describes who will have access to the memory allocated and + * with what permissions. + * @src_mem_type: The type of memory that will be used to satisfy the allocation. + * @src_data: A pointer to auxiliary data required to satisfy the allocation. + */ +void *mem_buf_construct_alloc_req(void *mem_buf_txn, size_t alloc_size, + struct gh_acl_desc *acl_desc, + enum mem_buf_mem_type src_mem_type, void *src_data) +{ + size_t tot_size, alloc_req_size, acl_desc_size; + void *req_buf, *arb_payload; + unsigned int nr_acl_entries = acl_desc->n_acl_entries; + struct mem_buf_alloc_req *req; + struct mem_buf_txn *txn = mem_buf_txn; + int txn_id = txn->txn_id; + + + alloc_req_size = offsetof(struct mem_buf_alloc_req, + acl_desc.acl_entries[nr_acl_entries]); + tot_size = alloc_req_size + + mem_buf_get_mem_type_alloc_req_size(src_mem_type); + + req_buf = kzalloc(tot_size, GFP_KERNEL); + if (!req_buf) + return ERR_PTR(-ENOMEM); + + req = req_buf; + req->hdr.txn_id = txn_id; + req->hdr.msg_type = MEM_BUF_ALLOC_REQ; + req->hdr.msg_size = tot_size; + req->size = alloc_size; + req->src_mem_type = src_mem_type; + acl_desc_size = offsetof(struct gh_acl_desc, + acl_entries[nr_acl_entries]); + memcpy(&req->acl_desc, acl_desc, acl_desc_size); + + arb_payload = req_buf + alloc_req_size; + mem_buf_populate_alloc_req_arb_payload(arb_payload, src_data, + src_mem_type); + + trace_send_alloc_req(req); + return req_buf; +} +EXPORT_SYMBOL(mem_buf_construct_alloc_req); + +/* + * mem_buf_construct_alloc_resp: Construct a response message to an allocation request. + * @req_msg: The request message that is being replied to. + * @alloc_ret: The return code of the allocation. + * @memparcel_hdl: The memparcel handle that corresponds to the memory that was allocated. + * @gh_rm_trans_type: The type of memory transfer associated with the response (i.e. donation, + * sharing, or lending). + */ +void *mem_buf_construct_alloc_resp(void *req_msg, s32 alloc_ret, + gh_memparcel_handle_t memparcel_hdl, int gh_rm_trans_type) +{ + struct mem_buf_alloc_req *req = req_msg; + struct mem_buf_alloc_resp *resp_msg = kzalloc(sizeof(*resp_msg), GFP_KERNEL); + + if (!resp_msg) + return ERR_PTR(-ENOMEM); + + resp_msg->hdr.txn_id = req->hdr.txn_id; + resp_msg->hdr.msg_type = MEM_BUF_ALLOC_RESP; + resp_msg->hdr.msg_size = sizeof(*resp_msg); + resp_msg->ret = alloc_ret; + resp_msg->hdl = memparcel_hdl; + resp_msg->gh_rm_trans_type = gh_rm_trans_type; + + return resp_msg; +} +EXPORT_SYMBOL(mem_buf_construct_alloc_resp); + +/* + * mem_buf_construct_relinquish_msg: Construct a relinquish message. + * @txn_id: The transaction ID that corresponds to the memory that is being relinquished. + * @memparcel_hdl: The memparcel that corresponds to the memory that is being relinquished. + */ +void *mem_buf_construct_relinquish_msg(u32 txn_id, gh_memparcel_handle_t memparcel_hdl) +{ + struct mem_buf_alloc_relinquish *relinquish_msg; + + relinquish_msg = kzalloc(sizeof(*relinquish_msg), GFP_KERNEL); + if (!relinquish_msg) + return ERR_PTR(-ENOMEM); + + relinquish_msg->hdr.msg_type = MEM_BUF_ALLOC_RELINQUISH; + relinquish_msg->hdr.msg_size = sizeof(*relinquish_msg); + relinquish_msg->hdr.txn_id = txn_id; + relinquish_msg->hdl = memparcel_hdl; + + return relinquish_msg; +} +EXPORT_SYMBOL(mem_buf_construct_relinquish_msg); + +int mem_buf_retrieve_txn_id(void *mem_buf_txn) +{ + struct mem_buf_txn *txn = mem_buf_txn; + + return txn->txn_id; +} +EXPORT_SYMBOL(mem_buf_retrieve_txn_id); + +/* + * mem_buf_init_txn: Allocates a mem-buf transaction that is used in request-response + * message pairs. + * @mem_buf_msgq_hdl: The handle for the message queue that will be used to transmit the message. + * @resp_buf: The buffer that will store the output of the response from the recipient of the + * request. + */ +void *mem_buf_init_txn(void *mem_buf_msgq_hdl, void *resp_buf) +{ + struct mem_buf_txn *txn; + struct mem_buf_msgq_desc *desc = mem_buf_msgq_hdl; + int ret; + + txn = kzalloc(sizeof(*txn), GFP_KERNEL); + if (!txn) + return ERR_PTR(-ENOMEM); + + mutex_lock(&desc->idr_mutex); + ret = idr_alloc_cyclic(&desc->txn_idr, txn, 0, INT_MAX, GFP_KERNEL); + mutex_unlock(&desc->idr_mutex); + if (ret < 0) { + pr_err("%s: failed to allocate transaction id rc: %d\n", __func__, ret); + kfree(txn); + return ERR_PTR(ret); + } + + txn->txn_id = ret; + init_completion(&txn->txn_done); + txn->resp_buf = resp_buf; + + return txn; +} +EXPORT_SYMBOL(mem_buf_init_txn); + +/* + * mem_buf_msgq_send: Send a mem-buf message over a particular message queue. + * @mem_buf_msgq_hdl: The handle for the message queue that will be used to send the message. + * @msg: The message to be sent. This message must be a mem-buf allocation request, response, or + * relinquish request. + */ +int mem_buf_msgq_send(void *mem_buf_msgq_hdl, void *msg) +{ + struct mem_buf_msgq_desc *desc = mem_buf_msgq_hdl; + struct mem_buf_msg_hdr *hdr = msg; + int ret; + + if (!(hdr->msg_type >= MEM_BUF_ALLOC_REQ && hdr->msg_type < MEM_BUF_ALLOC_REQ_MAX)) { + pr_err("%s: message type invalid\n"); + return -EINVAL; + } + + ret = gh_msgq_send(desc->msgq_hdl, msg, hdr->msg_size, 0); + if (ret < 0) + pr_err("%s: failed to send allocation request rc: %d\n", __func__, ret); + else + pr_debug("%s: alloc request sent\n", __func__); + + return ret; +} +EXPORT_SYMBOL(mem_buf_msgq_send); + +/* + * mem_buf_txn_wait: Wait for a response for a particular request. + * @mem_buf_txn: A valid transaction which corresponds to a request that was sent. + * + * When this function returns successfully, the output of the response will be in the @resp_buf + * parameter that was passed into mem_buf_txn_init(). + */ +int mem_buf_txn_wait(void *mem_buf_txn) +{ + struct mem_buf_txn *txn = mem_buf_txn; + int ret; + + pr_debug("%s: waiting for allocation response\n", __func__); + ret = wait_for_completion_timeout(&txn->txn_done, + msecs_to_jiffies(MEM_BUF_TIMEOUT_MS)); + if (ret == 0) { + pr_err("%s: timed out waiting for allocation response\n", + __func__); + return -ETIMEDOUT; + } + pr_debug("%s: alloc response received\n", __func__); + + return txn->txn_ret; +} +EXPORT_SYMBOL(mem_buf_txn_wait); + +/* + * mem_buf_destroy_txn: Releases all resources associated with a mem-buf transaction. + * @mem_buf_msgq_hdl: The handle that corresponds to the message queue used for messaging. + * @mem_buf_txn: The transaction structure that was involved in the messaging. + */ +void mem_buf_destroy_txn(void *mem_buf_msgq_hdl, void *mem_buf_txn) +{ + struct mem_buf_msgq_desc *desc = mem_buf_msgq_hdl; + struct mem_buf_txn *txn = mem_buf_txn; + + mutex_lock(&desc->idr_mutex); + idr_remove(&desc->txn_idr, txn->txn_id); + mutex_unlock(&desc->idr_mutex); + kfree(txn); +} +EXPORT_SYMBOL(mem_buf_destroy_txn); + +static void mem_buf_process_alloc_resp(struct mem_buf_msgq_desc *desc, void *buf, size_t size) +{ + struct mem_buf_msg_hdr *hdr = buf; + struct mem_buf_alloc_resp *alloc_resp = buf; + struct mem_buf_txn *txn; + + mutex_lock(&desc->idr_mutex); + txn = idr_find(&desc->txn_idr, hdr->txn_id); + if (!txn) { + pr_err("%s no txn associated with id: %d\n", __func__, hdr->txn_id); + /* + * If this was a legitimate allocation, we should let the + * allocator know that the memory is not in use, so that + * it can be reclaimed. + */ + if (!alloc_resp->ret) { + desc->msgq_ops->relinquish_memparcel_hdl(desc->hdlr_data, hdr->txn_id, + alloc_resp->hdl); + kfree(buf); + } + } else { + txn->txn_ret = desc->msgq_ops->alloc_resp_hdlr(desc->hdlr_data, buf, size, + txn->resp_buf); + complete(&txn->txn_done); + } + mutex_unlock(&desc->idr_mutex); +} + +static void mem_buf_process_msg(struct mem_buf_msgq_desc *desc, void *buf, size_t size) +{ + struct mem_buf_msg_hdr *hdr = buf; + + pr_debug("%s: mem-buf message received\n", __func__); + if (size < sizeof(*hdr) || hdr->msg_size != size) { + pr_err("%s: message received is not of a proper size: 0x%lx\n", + __func__, size); + kfree(buf); + return; + } + + switch (hdr->msg_type) { + case MEM_BUF_ALLOC_REQ: + desc->msgq_ops->alloc_req_hdlr(desc->hdlr_data, buf, size); + break; + case MEM_BUF_ALLOC_RESP: + mem_buf_process_alloc_resp(desc, buf, size); + break; + case MEM_BUF_ALLOC_RELINQUISH: + desc->msgq_ops->relinquish_hdlr(desc->hdlr_data, buf, size); + break; + default: + pr_err("%s: received message of unknown type: %d\n", __func__, + hdr->msg_type); + kfree(buf); + } +} + +static int mem_buf_msgq_name_to_msgq_label(const char *name) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(mem_buf_msgqs); i++) + if (!strcmp(name, mem_buf_msgqs[i].name)) + return mem_buf_msgqs[i].label; + + return -EINVAL; +} + +static int mem_buf_msgq_recv_fn(void *data) +{ + struct mem_buf_msgq_desc *desc = data; + void *buf; + size_t size; + int ret; + + while (!kthread_should_stop()) { + buf = kzalloc(GH_MSGQ_MAX_MSG_SIZE_BYTES, GFP_KERNEL); + if (!buf) + continue; + + ret = gh_msgq_recv(desc->msgq_hdl, buf, GH_MSGQ_MAX_MSG_SIZE_BYTES, &size, 0); + if (ret < 0) { + kfree(buf); + pr_err_ratelimited("%s failed to receive message rc: %d\n", __func__, ret); + } else { + mem_buf_process_msg(desc, buf, size); + } + } + + return 0; +} + +void *mem_buf_msgq_register(const char *msgq_name, struct mem_buf_msgq_hdlr_info *info) +{ + struct mem_buf_msgq_desc *desc = kzalloc(sizeof(*desc), GFP_KERNEL); + int label; + void *ret; + + if (!desc) + return ERR_PTR(-ENOMEM); + else if (!info || !info->msgq_ops || !info->msgq_ops->alloc_req_hdlr || + !info->msgq_ops->alloc_resp_hdlr || !info->msgq_ops->relinquish_hdlr) + return ERR_PTR(-EINVAL); + + label = mem_buf_msgq_name_to_msgq_label(msgq_name); + if (label < 0) + return ERR_PTR(label); + + INIT_LIST_HEAD(&desc->list); + desc->msgq_ops = info->msgq_ops; + desc->hdlr_data = info->hdlr_data; + mutex_init(&desc->idr_mutex); + idr_init(&desc->txn_idr); + + desc->msgq_hdl = gh_msgq_register(label); + if (IS_ERR(desc->msgq_hdl)) { + ret = desc->msgq_hdl; + pr_err("Message queue registration failed: rc: %d\n", PTR_ERR(desc->msgq_hdl)); + goto err_msgq_register; + } + + mutex_lock(&mem_buf_msgq_list_lock); + list_add_tail(&desc->list, &mem_buf_msgq_list); + mutex_unlock(&mem_buf_msgq_list_lock); + + desc->recv_thr = kthread_run(mem_buf_msgq_recv_fn, desc, "mem_buf_%s_rcvr", msgq_name); + if (IS_ERR(desc->recv_thr)) { + ret = desc->recv_thr; + pr_err("Failed to create msgq receiver thread rc: %d\n", PTR_ERR(desc->recv_thr)); + goto err_thr_create; + } + + return desc; + +err_thr_create: + gh_msgq_unregister(desc->msgq_hdl); +err_msgq_register: + idr_destroy(&desc->txn_idr); + mutex_destroy(&desc->idr_mutex); + kfree(desc); + return ret; +} +EXPORT_SYMBOL(mem_buf_msgq_register); + +void mem_buf_msgq_unregister(void *mem_buf_msgq_hdl) +{ + struct mem_buf_msgq_desc *desc = mem_buf_msgq_hdl; + + kthread_stop(desc->recv_thr); + mutex_lock(&mem_buf_msgq_list_lock); + list_del(&desc->list); + mutex_unlock(&mem_buf_msgq_list_lock); + gh_msgq_unregister(desc->msgq_hdl); + idr_destroy(&desc->txn_idr); + mutex_destroy(&desc->idr_mutex); + kfree(desc); +} +EXPORT_SYMBOL(mem_buf_msgq_unregister); + +MODULE_LICENSE("GPL"); diff --git a/drivers/soc/qcom/mem_buf/mem-buf-msgq.h b/drivers/soc/qcom/mem_buf/mem-buf-msgq.h new file mode 100644 index 000000000000..374fb3882403 --- /dev/null +++ b/drivers/soc/qcom/mem_buf/mem-buf-msgq.h @@ -0,0 +1,257 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef MEM_BUF_MSGQ_H +#define MEM_BUF_MSGQ_H + +#include +#include + +/** + * enum mem_buf_msg_type: Message types used by the membuf driver for + * communication. + * @MEM_BUF_ALLOC_REQ: The message is an allocation request from another VM to + * the receiving VM + * @MEM_BUF_ALLOC_RESP: The message is a response from a remote VM to an + * allocation request issued by the receiving VM + * @MEM_BUF_ALLOC_RELINQUISH: The message is a notification from another VM + * that the receiving VM can reclaim the memory. + */ +enum mem_buf_msg_type { + MEM_BUF_ALLOC_REQ, + MEM_BUF_ALLOC_RESP, + MEM_BUF_ALLOC_RELINQUISH, + MEM_BUF_ALLOC_REQ_MAX, +}; + +/** + * struct mem_buf_msg_hdr: The header for all membuf messages + * @txn_id: The transaction ID for the message. This field is only meaningful + * for request/response type of messages. + * @msg_type: The type of message. + * @msg_size: The size of message. + */ +struct mem_buf_msg_hdr { + u32 txn_id; + u32 msg_type; + u32 msg_size; +} __packed; + +/** + * struct mem_buf_alloc_req: The message format for a memory allocation request + * to another VM. + * @hdr: Message header + * @size: The size of the memory allocation to be performed on the remote VM. + * @src_mem_type: The type of memory that the remote VM should allocate. + * @acl_desc: A GH ACL descriptor that describes the VMIDs that will be + * accessing the memory, as well as what permissions each VMID will have. + * + * NOTE: Certain memory types require additional information for the remote VM + * to interpret. That information should be concatenated with this structure + * prior to sending the allocation request to the remote VM. For example, + * with memory type DMAHEAP, the allocation request message will consist of this + * structure, as well as the name of the heap that will source the allocation. + */ +struct mem_buf_alloc_req { + struct mem_buf_msg_hdr hdr; + u64 size; + u32 src_mem_type; + struct gh_acl_desc acl_desc; +} __packed; + +/** + * struct mem_buf_alloc_resp: The message format for a memory allocation + * request response. + * @hdr: Message header + * @ret: Return code from remote VM + * @hdl: The memparcel handle associated with the memory allocated to the + * receiving VM. This field is only meaningful if the allocation on the remote + * VM was carried out successfully, as denoted by @ret. + * @gh_rm_trans_type: Denotes the type of memory transfer associated with the response + * (i.e. memory donation, sharing, or lending). + */ +struct mem_buf_alloc_resp { + struct mem_buf_msg_hdr hdr; + s32 ret; + u32 hdl; + int gh_rm_trans_type; +} __packed; + +/** + * struct mem_buf_alloc_relinquish: The message format for a notification + * that the current VM has relinquished access to the memory lent to it by + * another VM. + * @hdr: Message header + * @hdl: The memparcel handle associated with the memory. + */ +struct mem_buf_alloc_relinquish { + struct mem_buf_msg_hdr hdr; + u32 hdl; +} __packed; + +/* + * mem_buf_msgq_ops: A set of ops that are invoked when a message of particular + * types are received by a mem-buf message queue. + * + * The handlers are responsible for freeing the msg buffer that is passed to + * them. + * + * @alloc_req_hdlr: The handler for messages of type MEM_BUF_ALLOC_REQ. + * @alloc_resp_hdlr: The handler for messages of type MEM_BUF_ALLOC_RESP. + * @relinquish_hdlr: The handler for messages of type MEM_BUF_ALLOC_RELINQUISH. + * @relinquish_memparcel_hdl: Callback for relinquishing a memparcel. This is typically used in + * case an allocation request times out, and the response arrives late. + * In this case, the transaction will have been aborted, but the memory + * still needs to be relinquished. + */ +struct mem_buf_msgq_ops { + void (*alloc_req_hdlr)(void *hdlr_data, void *msg, size_t size); + int (*alloc_resp_hdlr)(void *hdlr_data, void *msg, size_t size, void *resp_buf); + void (*relinquish_hdlr)(void *hdlr_data, void *msg, size_t size); + void (*relinquish_memparcel_hdl)(void *hdlr_data, u32 txn_id, + gh_memparcel_handle_t memparcel_hdl); +}; + +struct mem_buf_msgq_hdlr_info { + const struct mem_buf_msgq_ops *msgq_ops; + void *hdlr_data; +}; + +static inline u32 get_alloc_req_nr_acl_entries(struct mem_buf_alloc_req *req) +{ + return req->acl_desc.n_acl_entries; +} + +static inline struct gh_acl_desc *get_alloc_req_gh_acl_desc(struct mem_buf_alloc_req *req) +{ + return &req->acl_desc; +} + +static inline enum mem_buf_mem_type get_alloc_req_src_mem_type(struct mem_buf_alloc_req *req) +{ + return req->src_mem_type; +} + +static inline u64 get_alloc_req_size(struct mem_buf_alloc_req *req) +{ + return req->size; +} + +static inline void *get_alloc_req_arb_payload(struct mem_buf_alloc_req *req) +{ + void *buf = req; + size_t nr_acl_entries; + size_t payload_offset; + + nr_acl_entries = req->acl_desc.n_acl_entries; + if (nr_acl_entries != 1) + return NULL; + + payload_offset = offsetof(struct mem_buf_alloc_req, + acl_desc.acl_entries[nr_acl_entries]); + + return buf + payload_offset; +} + +static inline u32 get_alloc_req_txn_id(struct mem_buf_alloc_req *req) +{ + return req->hdr.txn_id; +} + +static inline s32 get_alloc_resp_retval(struct mem_buf_alloc_resp *resp) +{ + return resp->ret; +} + +static inline u32 get_alloc_resp_hdl(struct mem_buf_alloc_resp *resp) +{ + return resp->hdl; +} + +static inline int get_alloc_resp_trans_type(struct mem_buf_alloc_resp *resp) +{ + return resp->gh_rm_trans_type; +} + +static inline u32 get_relinquish_req_txn_id(struct mem_buf_alloc_relinquish *relinquish_msg) +{ + return relinquish_msg->hdr.txn_id; +} + +#if IS_ENABLED(CONFIG_QCOM_MEM_BUF_MSGQ) +void *mem_buf_msgq_register(const char *msgq_name, struct mem_buf_msgq_hdlr_info *info); +void mem_buf_msgq_unregister(void *mem_buf_msgq_hdl); +void *mem_buf_init_txn(void *mem_buf_msgq_hdl, void *resp_buf); +int mem_buf_msgq_send(void *mem_buf_msgq_hdl, void *msg); +int mem_buf_txn_wait(void *mem_buf_txn); +void mem_buf_destroy_txn(void *mem_buf_msgq_hdl, void *mem_buf_txn); +int mem_buf_retrieve_txn_id(void *mem_buf_txn); +/* + * It is the caller's responsibility to free the messages returned by + * any functions invoked to construct them via a call to kfree(). + */ +void *mem_buf_construct_alloc_req(void *mem_buf_txn, size_t alloc_size, + struct gh_acl_desc *acl_desc, + enum mem_buf_mem_type src_mem_type, void *src_data); +void *mem_buf_construct_alloc_resp(void *req_msg, s32 alloc_ret, + gh_memparcel_handle_t memparcel_hdl, int gh_rm_trans_type); +void *mem_buf_construct_relinquish_msg(u32 txn_id, gh_memparcel_handle_t memparcel_hdl); +#else +static inline void *mem_buf_msgq_register(const char *msgq_name, + struct mem_buf_msgq_hdlr_info *info) +{ + return ERR_PTR(-EOPNOTSUPP); +} + +static inline void mem_buf_msgq_unregister(void *mem_buf_msgq_hdl) +{ +} + +static inline void *mem_buf_init_txn(void *mem_buf_msgq_hdl, void *resp_buf) +{ + return ERR_PTR(-EOPNOTSUPP); +} + +static inline int mem_buf_msgq_send(void *mem_buf_msgq_hdl, void *msg) +{ + return -EOPNOTSUPP; +} + +static inline int mem_buf_txn_wait(void *mem_buf_txn) +{ + return -EOPNOTSUPP; +} + +static inline void mem_buf_destroy_txn(void *mem_buf_msgq_hdl, void *mem_buf_txn) +{ +} + +static inline void *mem_buf_construct_alloc_req(void *mem_buf_txn, size_t alloc_size, + struct gh_acl_desc *acl_desc, + enum mem_buf_mem_type src_mem_type, void *src_data) + +{ + return ERR_PTR(-ENODEV); +} + +static inline void *mem_buf_construct_alloc_resp(void *req_msg, s32 alloc_ret, + gh_memparcel_handle_t memparcel_hdl, int gh_rm_trans_type) +{ + return ERR_PTR(-ENODEV); +} + +static inline void *mem_buf_construct_relinquish_msg(u32 txn_id, + gh_memparcel_handle_t memparcel_hdl) +{ + return ERR_PTR(-ENODEV); +} + +static inline int mem_buf_retrieve_txn_id(void *mem_buf_txn) +{ + return -ENODEV; +} +#endif +#endif diff --git a/drivers/soc/qcom/mem_buf/mem-buf.c b/drivers/soc/qcom/mem_buf/mem-buf.c new file mode 100644 index 000000000000..42dca0451be9 --- /dev/null +++ b/drivers/soc/qcom/mem_buf/mem-buf.c @@ -0,0 +1,392 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mem-buf-gh.h" +#include "mem-buf-ids.h" + +#define MEM_BUF_MAX_DEVS 1 +static dev_t mem_buf_dev_no; +static struct class *mem_buf_class; +static struct cdev mem_buf_char_dev; + +union mem_buf_ioctl_arg { + struct mem_buf_alloc_ioctl_arg allocation; + struct mem_buf_lend_ioctl_arg lend; + struct mem_buf_retrieve_ioctl_arg retrieve; + struct mem_buf_reclaim_ioctl_arg reclaim; + struct mem_buf_share_ioctl_arg share; + struct mem_buf_exclusive_owner_ioctl_arg get_ownership; +}; + +static bool is_valid_mem_buf_perms(u32 mem_buf_perms) +{ + if (mem_buf_perms & ~MEM_BUF_PERM_VALID_FLAGS) { + pr_err_ratelimited("%s: Invalid mem-buf permissions detected\n", + __func__); + return false; + } + return true; +} + +static int mem_buf_perms_to_perms(u32 mem_buf_perms) +{ + int perms = 0; + + if (!is_valid_mem_buf_perms(mem_buf_perms)) + return -EINVAL; + + if (mem_buf_perms & MEM_BUF_PERM_FLAG_READ) + perms |= PERM_READ; + if (mem_buf_perms & MEM_BUF_PERM_FLAG_WRITE) + perms |= PERM_WRITE; + if (mem_buf_perms & MEM_BUF_PERM_FLAG_EXEC) + perms |= PERM_EXEC; + + return perms; +} + +int mem_buf_acl_to_vmid_perms_list(unsigned int nr_acl_entries, const void __user *acl_entries, + int **dst_vmids, int **dst_perms) +{ + int ret, i, *vmids, *perms; + struct acl_entry entry; + + if (!nr_acl_entries || !acl_entries) + return -EINVAL; + + vmids = kmalloc_array(nr_acl_entries, sizeof(*vmids), GFP_KERNEL); + if (!vmids) + return -ENOMEM; + + perms = kmalloc_array(nr_acl_entries, sizeof(*perms), GFP_KERNEL); + if (!perms) { + kfree(vmids); + return -ENOMEM; + } + + for (i = 0; i < nr_acl_entries; i++) { + ret = copy_struct_from_user(&entry, sizeof(entry), + acl_entries + (sizeof(entry) * i), + sizeof(entry)); + if (ret < 0) + goto out; + + vmids[i] = mem_buf_fd_to_vmid(entry.vmid); + perms[i] = mem_buf_perms_to_perms(entry.perms); + if (vmids[i] < 0 || perms[i] < 0) { + ret = -EINVAL; + goto out; + } + } + + *dst_vmids = vmids; + *dst_perms = perms; + return ret; + +out: + kfree(perms); + kfree(vmids); + return ret; +} + +static int mem_buf_lend_user(struct mem_buf_lend_ioctl_arg *uarg, bool is_lend) +{ + int *vmids, *perms; + int ret; + struct dma_buf *dmabuf; + struct mem_buf_lend_kernel_arg karg = {0}; + + if (!uarg->nr_acl_entries || !uarg->acl_list || + uarg->nr_acl_entries > MEM_BUF_MAX_NR_ACL_ENTS || + uarg->reserved0 || uarg->reserved1 || uarg->reserved2) + return -EINVAL; + + dmabuf = dma_buf_get(uarg->dma_buf_fd); + if (IS_ERR(dmabuf)) + return PTR_ERR(dmabuf); + + ret = mem_buf_acl_to_vmid_perms_list(uarg->nr_acl_entries, + (void *)uarg->acl_list, &vmids, &perms); + if (ret) + goto err_acl; + + karg.nr_acl_entries = uarg->nr_acl_entries; + karg.vmids = vmids; + karg.perms = perms; + + if (is_lend) { + ret = mem_buf_lend(dmabuf, &karg); + if (ret) + goto err_lend; + } else { + ret = mem_buf_share(dmabuf, &karg); + if (ret) + goto err_lend; + } + + uarg->memparcel_hdl = karg.memparcel_hdl; +err_lend: + kfree(perms); + kfree(vmids); +err_acl: + dma_buf_put(dmabuf); + return ret; +} + +static int mem_buf_reclaim_user(struct mem_buf_reclaim_ioctl_arg *uarg) +{ + struct dma_buf *dmabuf; + int ret; + + if (uarg->reserved0 || uarg->reserved1 || uarg->reserved2) + return -EINVAL; + + dmabuf = dma_buf_get(uarg->dma_buf_fd); + if (IS_ERR(dmabuf)) + return PTR_ERR(dmabuf); + + ret = mem_buf_reclaim(dmabuf); + dma_buf_put(dmabuf); + return ret; +} + +static int mem_buf_get_exclusive_ownership(struct mem_buf_exclusive_owner_ioctl_arg *uarg) +{ + struct dma_buf *dmabuf; + int ret = 0; + + dmabuf = dma_buf_get(uarg->dma_buf_fd); + if (IS_ERR(dmabuf)) + return PTR_ERR(dmabuf); + + if (IS_ERR(to_mem_buf_vmperm(dmabuf))) { + ret = -EINVAL; + goto put_dma_buf; + } + + uarg->is_exclusive_owner = mem_buf_dma_buf_exclusive_owner(dmabuf); + +put_dma_buf: + dma_buf_put(dmabuf); + + return ret; +} + +static long mem_buf_dev_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg) +{ + int fd; + unsigned int dir = _IOC_DIR(cmd); + union mem_buf_ioctl_arg ioctl_arg; + + if (_IOC_SIZE(cmd) > sizeof(ioctl_arg)) + return -EINVAL; + + if (copy_from_user(&ioctl_arg, (void __user *)arg, _IOC_SIZE(cmd))) + return -EFAULT; + + if (!(dir & _IOC_WRITE)) + memset(&ioctl_arg, 0, sizeof(ioctl_arg)); + + switch (cmd) { + case MEM_BUF_IOC_ALLOC: + { + struct mem_buf_alloc_ioctl_arg *allocation = + &ioctl_arg.allocation; + + if (!(mem_buf_capability & MEM_BUF_CAP_CONSUMER)) + return -EOPNOTSUPP; + + fd = mem_buf_alloc_fd(allocation); + + if (fd < 0) + return fd; + + allocation->mem_buf_fd = fd; + break; + } + case MEM_BUF_IOC_LEND: + { + struct mem_buf_lend_ioctl_arg *lend = &ioctl_arg.lend; + int ret; + + ret = mem_buf_lend_user(lend, true); + if (ret) + return ret; + + break; + } + case MEM_BUF_IOC_RETRIEVE: + { + struct mem_buf_retrieve_ioctl_arg *retrieve = + &ioctl_arg.retrieve; + int ret; + + ret = mem_buf_retrieve_user(retrieve); + if (ret) + return ret; + break; + } + case MEM_BUF_IOC_RECLAIM: + { + struct mem_buf_reclaim_ioctl_arg *reclaim = + &ioctl_arg.reclaim; + int ret; + + ret = mem_buf_reclaim_user(reclaim); + if (ret) + return ret; + break; + } + case MEM_BUF_IOC_SHARE: + { + struct mem_buf_share_ioctl_arg *share = &ioctl_arg.share; + int ret; + + /* The two formats are currently identical */ + ret = mem_buf_lend_user((struct mem_buf_lend_ioctl_arg *)share, + false); + if (ret) + return ret; + + break; + } + case MEM_BUF_IOC_EXCLUSIVE_OWNER: + { + struct mem_buf_exclusive_owner_ioctl_arg *get_ownership = &ioctl_arg.get_ownership; + int ret; + + ret = mem_buf_get_exclusive_ownership(get_ownership); + if (ret) + return ret; + + break; + } + + default: + return -ENOTTY; + } + + if (dir & _IOC_READ) { + if (copy_to_user((void __user *)arg, &ioctl_arg, + _IOC_SIZE(cmd))) + return -EFAULT; + } + + return 0; +} + +static const struct file_operations mem_buf_dev_fops = { + .unlocked_ioctl = mem_buf_dev_ioctl, + .compat_ioctl = compat_ptr_ioctl, +}; + +static int mem_buf_msgq_probe(struct platform_device *pdev) +{ + int ret; + struct device *dev = &pdev->dev; + struct device *class_dev; + + if (!mem_buf_dev) + return -EPROBE_DEFER; + + ret = mem_buf_msgq_alloc(dev); + if (ret) + return ret; + + cdev_init(&mem_buf_char_dev, &mem_buf_dev_fops); + ret = cdev_add(&mem_buf_char_dev, mem_buf_dev_no, MEM_BUF_MAX_DEVS); + if (ret < 0) + goto err_cdev_add; + + class_dev = device_create(mem_buf_class, NULL, mem_buf_dev_no, NULL, + "membuf"); + if (IS_ERR(class_dev)) { + ret = PTR_ERR(class_dev); + goto err_dev_create; + } + + return 0; + +err_dev_create: + cdev_del(&mem_buf_char_dev); +err_cdev_add: + mem_buf_msgq_free(dev); + return ret; +} + +static int mem_buf_msgq_remove(struct platform_device *pdev) +{ + device_destroy(mem_buf_class, mem_buf_dev_no); + cdev_del(&mem_buf_char_dev); + mem_buf_msgq_free(&pdev->dev); + return 0; +} + +static const struct of_device_id mem_buf_msgq_match_tbl[] = { + {.compatible = "qcom,mem-buf-msgq"}, + {}, +}; + +static struct platform_driver mem_buf_msgq_driver = { + .probe = mem_buf_msgq_probe, + .remove = mem_buf_msgq_remove, + .driver = { + .name = "mem-buf-msgq", + .of_match_table = of_match_ptr(mem_buf_msgq_match_tbl), + }, +}; + +static int __init mem_buf_init(void) +{ + int ret; + + ret = alloc_chrdev_region(&mem_buf_dev_no, 0, MEM_BUF_MAX_DEVS, + "membuf"); + if (ret < 0) + goto err_chrdev_region; + + mem_buf_class = class_create(THIS_MODULE, "membuf"); + if (IS_ERR(mem_buf_class)) { + ret = PTR_ERR(mem_buf_class); + goto err_class_create; + } + + ret = platform_driver_register(&mem_buf_msgq_driver); + if (ret < 0) + goto err_platform_drvr_register; + + return 0; + +err_platform_drvr_register: + class_destroy(mem_buf_class); +err_class_create: + unregister_chrdev_region(mem_buf_dev_no, MEM_BUF_MAX_DEVS); +err_chrdev_region: + return ret; +} +module_init(mem_buf_init); + +static void __exit mem_buf_exit(void) +{ + platform_driver_unregister(&mem_buf_msgq_driver); + class_destroy(mem_buf_class); + unregister_chrdev_region(mem_buf_dev_no, MEM_BUF_MAX_DEVS); +} +module_exit(mem_buf_exit); + +MODULE_DESCRIPTION("Qualcomm Technologies, Inc. Memory Buffer Sharing driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/soc/qcom/mem_buf/mem_buf_dma_buf.c b/drivers/soc/qcom/mem_buf/mem_buf_dma_buf.c new file mode 100644 index 000000000000..56deb4d8ffc9 --- /dev/null +++ b/drivers/soc/qcom/mem_buf/mem_buf_dma_buf.c @@ -0,0 +1,681 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) "mem_buf_vmperm: " fmt + +#include +#include +#include "mem-buf-dev.h" +#include "mem-buf-gh.h" +#include "mem-buf-ids.h" + +struct mem_buf_vmperm { + u32 flags; + int current_vm_perms; + u32 mapcount; + int *vmids; + int *perms; + unsigned int nr_acl_entries; + unsigned int max_acl_entries; + struct dma_buf *dmabuf; + struct sg_table *sgt; + gh_memparcel_handle_t memparcel_hdl; + struct mutex lock; + mem_buf_dma_buf_destructor dtor; + void *dtor_data; +}; + +/* + * Ensures the vmperm can hold at least nr_acl_entries. + * Caller must hold vmperm->lock. + */ +static int mem_buf_vmperm_resize(struct mem_buf_vmperm *vmperm, + u32 new_size) +{ + int *vmids_copy, *perms_copy, *vmids, *perms; + u32 old_size; + + old_size = vmperm->max_acl_entries; + if (old_size >= new_size) + return 0; + + vmids = vmperm->vmids; + perms = vmperm->perms; + vmids_copy = kcalloc(new_size, sizeof(*vmids), GFP_KERNEL); + if (!vmids_copy) + return -ENOMEM; + perms_copy = kcalloc(new_size, sizeof(*perms), GFP_KERNEL); + if (!perms_copy) + goto out_perms; + + if (vmperm->vmids) { + memcpy(vmids_copy, vmids, sizeof(*vmids) * old_size); + kfree(vmids); + } + if (vmperm->perms) { + memcpy(perms_copy, perms, sizeof(*perms) * old_size); + kfree(perms); + } + vmperm->vmids = vmids_copy; + vmperm->perms = perms_copy; + vmperm->max_acl_entries = new_size; + return 0; + +out_perms: + kfree(vmids_copy); + return -ENOMEM; +} + +/* + * Caller should hold vmperm->lock. + */ +static void mem_buf_vmperm_update_state(struct mem_buf_vmperm *vmperm, int *vmids, + int *perms, u32 nr_acl_entries) +{ + int i; + size_t size = sizeof(*vmids) * nr_acl_entries; + + WARN_ON(vmperm->max_acl_entries < nr_acl_entries); + + memcpy(vmperm->vmids, vmids, size); + memcpy(vmperm->perms, perms, size); + vmperm->nr_acl_entries = nr_acl_entries; + + vmperm->current_vm_perms = 0; + for (i = 0; i < nr_acl_entries; i++) { + if (vmids[i] == current_vmid) + vmperm->current_vm_perms = perms[i]; + } +} + +/* + * Some types of errors may leave the memory in an unknown state. + * Since we cannot guarantee that accessing this memory is safe, + * acquire an extra reference count to the underlying dmabuf to + * prevent it from being freed. + * If this error occurs during dma_buf_release(), the file refcount + * will already be zero. In this case handling the error is the caller's + * responsibility. + */ +static void mem_buf_vmperm_set_err(struct mem_buf_vmperm *vmperm) +{ + get_dma_buf(vmperm->dmabuf); + vmperm->flags |= MEM_BUF_WRAPPER_FLAG_ERR; +} + +static struct mem_buf_vmperm *mem_buf_vmperm_alloc_flags( + struct sg_table *sgt, u32 flags, + int *vmids, int *perms, u32 nr_acl_entries) +{ + struct mem_buf_vmperm *vmperm; + int ret; + + vmperm = kzalloc(sizeof(*vmperm), GFP_KERNEL); + if (!vmperm) + return ERR_PTR(-ENOMEM); + + mutex_init(&vmperm->lock); + mutex_lock(&vmperm->lock); + ret = mem_buf_vmperm_resize(vmperm, nr_acl_entries); + if (ret) + goto err_resize_state; + + mem_buf_vmperm_update_state(vmperm, vmids, perms, + nr_acl_entries); + mutex_unlock(&vmperm->lock); + vmperm->sgt = sgt; + vmperm->flags = flags; + vmperm->memparcel_hdl = MEM_BUF_MEMPARCEL_INVALID; + + return vmperm; + +err_resize_state: + mutex_unlock(&vmperm->lock); + kfree(vmperm); + return ERR_PTR(-ENOMEM); +} + +/* Must be freed via mem_buf_vmperm_release. */ +struct mem_buf_vmperm *mem_buf_vmperm_alloc_accept(struct sg_table *sgt, + gh_memparcel_handle_t memparcel_hdl) +{ + int vmids[1]; + int perms[1]; + struct mem_buf_vmperm *vmperm; + + vmids[0] = current_vmid; + perms[0] = PERM_READ | PERM_WRITE | PERM_EXEC; + vmperm = mem_buf_vmperm_alloc_flags(sgt, + MEM_BUF_WRAPPER_FLAG_ACCEPT, + vmids, perms, 1); + if (IS_ERR(vmperm)) + return vmperm; + + vmperm->memparcel_hdl = memparcel_hdl; + return vmperm; +} +EXPORT_SYMBOL(mem_buf_vmperm_alloc_accept); + +struct mem_buf_vmperm *mem_buf_vmperm_alloc_staticvm(struct sg_table *sgt, + int *vmids, int *perms, u32 nr_acl_entries) +{ + return mem_buf_vmperm_alloc_flags(sgt, + MEM_BUF_WRAPPER_FLAG_STATIC_VM, + vmids, perms, nr_acl_entries); +} +EXPORT_SYMBOL(mem_buf_vmperm_alloc_staticvm); + +struct mem_buf_vmperm *mem_buf_vmperm_alloc(struct sg_table *sgt) +{ + int vmids[1]; + int perms[1]; + + vmids[0] = current_vmid; + perms[0] = PERM_READ | PERM_WRITE | PERM_EXEC; + return mem_buf_vmperm_alloc_flags(sgt, 0, + vmids, perms, 1); +} +EXPORT_SYMBOL(mem_buf_vmperm_alloc); + +static int __mem_buf_vmperm_reclaim(struct mem_buf_vmperm *vmperm) +{ + int ret; + int new_vmids[] = {current_vmid}; + int new_perms[] = {PERM_READ | PERM_WRITE | PERM_EXEC}; + + ret = mem_buf_unassign_mem(vmperm->sgt, vmperm->vmids, + vmperm->nr_acl_entries, + vmperm->memparcel_hdl); + if (ret) { + pr_err_ratelimited("Reclaim failed\n"); + mem_buf_vmperm_set_err(vmperm); + return ret; + } + + mem_buf_vmperm_update_state(vmperm, new_vmids, new_perms, 1); + vmperm->flags &= ~MEM_BUF_WRAPPER_FLAG_LENDSHARE; + vmperm->memparcel_hdl = MEM_BUF_MEMPARCEL_INVALID; + return 0; +} + +static int mem_buf_vmperm_relinquish(struct mem_buf_vmperm *vmperm) +{ + int ret; + struct gh_sgl_desc *sgl_desc; + + sgl_desc = mem_buf_sgt_to_gh_sgl_desc(vmperm->sgt); + if (IS_ERR(sgl_desc)) + return PTR_ERR(sgl_desc); + + ret = mem_buf_unmap_mem_s1(sgl_desc); + kvfree(sgl_desc); + if (ret) + return ret; + + ret = mem_buf_unmap_mem_s2(vmperm->memparcel_hdl); + return ret; +} + +int mem_buf_vmperm_release(struct mem_buf_vmperm *vmperm) +{ + int ret = 0; + + if (vmperm->dtor) { + ret = vmperm->dtor(vmperm->dtor_data); + if (ret) + goto exit; + } + + mutex_lock(&vmperm->lock); + if (vmperm->flags & MEM_BUF_WRAPPER_FLAG_LENDSHARE) + ret = __mem_buf_vmperm_reclaim(vmperm); + else if (vmperm->flags & MEM_BUF_WRAPPER_FLAG_ACCEPT) + ret = mem_buf_vmperm_relinquish(vmperm); + + mutex_unlock(&vmperm->lock); +exit: + kfree(vmperm->perms); + kfree(vmperm->vmids); + mutex_destroy(&vmperm->lock); + kfree(vmperm); + return ret; +} +EXPORT_SYMBOL(mem_buf_vmperm_release); + +int mem_buf_dma_buf_attach(struct dma_buf *dmabuf, struct dma_buf_attachment *attachment) +{ + struct mem_buf_dma_buf_ops *ops; + + ops = container_of(dmabuf->ops, struct mem_buf_dma_buf_ops, dma_ops); + return ops->attach(dmabuf, attachment); +} +EXPORT_SYMBOL(mem_buf_dma_buf_attach); + +struct mem_buf_vmperm *to_mem_buf_vmperm(struct dma_buf *dmabuf) +{ + struct mem_buf_dma_buf_ops *ops; + + if (dmabuf->ops->attach != mem_buf_dma_buf_attach) + return ERR_PTR(-EINVAL); + + ops = container_of(dmabuf->ops, struct mem_buf_dma_buf_ops, dma_ops); + return ops->lookup(dmabuf); +} +EXPORT_SYMBOL(to_mem_buf_vmperm); + +int mem_buf_dma_buf_set_destructor(struct dma_buf *buf, + mem_buf_dma_buf_destructor dtor, + void *dtor_data) +{ + struct mem_buf_vmperm *vmperm = to_mem_buf_vmperm(buf); + + if (IS_ERR(vmperm)) + return PTR_ERR(vmperm); + + vmperm->dtor = dtor; + vmperm->dtor_data = dtor_data; + + return 0; +} +EXPORT_SYMBOL(mem_buf_dma_buf_set_destructor); + +/* + * With CFI enabled, ops->attach must be set from *this* modules in order + * for the comparison test in to_mem_buf_vmperm() to work. + */ +struct dma_buf * +mem_buf_dma_buf_export(struct dma_buf_export_info *exp_info, + struct mem_buf_dma_buf_ops *ops) +{ + struct mem_buf_vmperm *vmperm; + struct dma_buf *dmabuf; + struct dma_buf_ops *dma_ops = &ops->dma_ops; + + if (dma_ops->attach != mem_buf_dma_buf_attach) { + if (!dma_ops->attach) { + dma_ops->attach = mem_buf_dma_buf_attach; + } else { + pr_err("Attach callback must be null! %ps\n", exp_info->ops); + return ERR_PTR(-EINVAL); + } + } + exp_info->ops = dma_ops; + + dmabuf = dma_buf_export(exp_info); + if (IS_ERR(dmabuf)) + return dmabuf; + + vmperm = to_mem_buf_vmperm(dmabuf); + if (WARN_ON(IS_ERR(vmperm))) { + dma_buf_put(dmabuf); + return ERR_PTR(-EINVAL); + } + + vmperm->dmabuf = dmabuf; + return dmabuf; +} +EXPORT_SYMBOL(mem_buf_dma_buf_export); + +void mem_buf_vmperm_pin(struct mem_buf_vmperm *vmperm) +{ + mutex_lock(&vmperm->lock); + vmperm->mapcount++; + mutex_unlock(&vmperm->lock); +} +EXPORT_SYMBOL(mem_buf_vmperm_pin); + +void mem_buf_vmperm_unpin(struct mem_buf_vmperm *vmperm) +{ + mutex_lock(&vmperm->lock); + if (!WARN_ON(vmperm->mapcount == 0)) + vmperm->mapcount--; + mutex_unlock(&vmperm->lock); +} +EXPORT_SYMBOL(mem_buf_vmperm_unpin); + +/* + * DC IVAC requires write permission, so no CMO on read-only buffers. + * We allow mapping to iommu regardless of permissions. + * Caller must have previously called mem_buf_vmperm_pin + */ +bool mem_buf_vmperm_can_cmo(struct mem_buf_vmperm *vmperm) +{ + u32 perms = PERM_READ | PERM_WRITE; + bool ret = false; + + mutex_lock(&vmperm->lock); + if (((vmperm->current_vm_perms & perms) == perms) && vmperm->mapcount) + ret = true; + mutex_unlock(&vmperm->lock); + return ret; +} +EXPORT_SYMBOL(mem_buf_vmperm_can_cmo); + +bool mem_buf_vmperm_can_mmap(struct mem_buf_vmperm *vmperm, struct vm_area_struct *vma) +{ + bool ret = false; + + mutex_lock(&vmperm->lock); + if (!vmperm->mapcount) + goto unlock; + if (!(vmperm->current_vm_perms & PERM_READ)) + goto unlock; + if (!(vmperm->current_vm_perms & PERM_WRITE) && + vma->vm_flags & VM_WRITE) + goto unlock; + if (!(vmperm->current_vm_perms & PERM_EXEC) && + vma->vm_flags & VM_EXEC) + goto unlock; + ret = true; +unlock: + mutex_unlock(&vmperm->lock); + return ret; +} +EXPORT_SYMBOL(mem_buf_vmperm_can_mmap); + +bool mem_buf_vmperm_can_vmap(struct mem_buf_vmperm *vmperm) +{ + u32 perms = PERM_READ | PERM_WRITE; + bool ret = false; + + mutex_lock(&vmperm->lock); + if (((vmperm->current_vm_perms & perms) == perms) && vmperm->mapcount) + ret = true; + mutex_unlock(&vmperm->lock); + return ret; +} +EXPORT_SYMBOL(mem_buf_vmperm_can_vmap); + +static int validate_lend_vmids(struct mem_buf_lend_kernel_arg *arg, + int op) +{ + int i; + bool found = false; + + for (i = 0; i < arg->nr_acl_entries; i++) { + if (arg->vmids[i] == current_vmid) { + found = true; + break; + } + } + + if (found && op == GH_RM_TRANS_TYPE_LEND) { + pr_err_ratelimited("Lend cannot target the current VM\n"); + return -EINVAL; + } else if (!found && op == GH_RM_TRANS_TYPE_SHARE) { + pr_err_ratelimited("Share must target the current VM\n"); + return -EINVAL; + } + return 0; +} + +/* + * Allow sharing buffers which are not mapped by either mmap, vmap, or dma. + * Also allow sharing mapped buffers if the new S2 permissions are at least + * as permissive as the old S2 permissions. Currently differences in + * executable permission are ignored, under the assumption the memory will + * not be used for this purpose. + */ +static bool validate_lend_mapcount(struct mem_buf_vmperm *vmperm, + struct mem_buf_lend_kernel_arg *arg) +{ + int i; + int perms = PERM_READ | PERM_WRITE; + + if (!vmperm->mapcount) + return true; + + for (i = 0; i < arg->nr_acl_entries; i++) { + if (arg->vmids[i] == current_vmid && + (arg->perms[i] & perms) == perms) + return true; + } + + pr_err_ratelimited("%s: dma-buf is pinned, dumping permissions!\n", __func__); + for (i = 0; i < arg->nr_acl_entries; i++) + pr_err_ratelimited("%s: VMID=%d PERM=%d\n", __func__, + arg->vmids[i], arg->perms[i]); + return false; +} + +static int mem_buf_lend_internal(struct dma_buf *dmabuf, + struct mem_buf_lend_kernel_arg *arg, + int op) +{ + struct mem_buf_vmperm *vmperm; + struct sg_table *sgt; + int ret; + + if (!arg->nr_acl_entries || !arg->vmids || !arg->perms) + return -EINVAL; + + vmperm = to_mem_buf_vmperm(dmabuf); + if (IS_ERR(vmperm)) { + pr_err_ratelimited("dmabuf ops %ps are not a mem_buf_dma_buf_ops\n", + dmabuf->ops); + return -EINVAL; + } + sgt = vmperm->sgt; + + ret = validate_lend_vmids(arg, op); + if (ret) + return ret; + + mutex_lock(&vmperm->lock); + if (vmperm->flags & MEM_BUF_WRAPPER_FLAG_STATIC_VM) { + pr_err_ratelimited("dma-buf is staticvm type!\n"); + mutex_unlock(&vmperm->lock); + return -EINVAL; + } + + if (vmperm->flags & MEM_BUF_WRAPPER_FLAG_LENDSHARE) { + pr_err_ratelimited("dma-buf already lent or shared!\n"); + mutex_unlock(&vmperm->lock); + return -EINVAL; + } + + if (vmperm->flags & MEM_BUF_WRAPPER_FLAG_ACCEPT) { + pr_err_ratelimited("dma-buf not owned by current vm!\n"); + mutex_unlock(&vmperm->lock); + return -EINVAL; + } + + if (!validate_lend_mapcount(vmperm, arg)) { + mutex_unlock(&vmperm->lock); + return -EINVAL; + } + + /* + * Although it would be preferrable to require clients to decide + * whether they require cache maintenance prior to caling this function + * for backwards compatibility with ion we will always do CMO. + */ + dma_map_sgtable(mem_buf_dev, vmperm->sgt, DMA_TO_DEVICE, 0); + dma_unmap_sgtable(mem_buf_dev, vmperm->sgt, DMA_TO_DEVICE, 0); + + ret = mem_buf_vmperm_resize(vmperm, arg->nr_acl_entries); + if (ret) + goto err_resize; + + ret = mem_buf_assign_mem(op, vmperm->sgt, arg); + if (ret) { + if (ret == -EADDRNOTAVAIL) + mem_buf_vmperm_set_err(vmperm); + goto err_assign; + } + + mem_buf_vmperm_update_state(vmperm, arg->vmids, arg->perms, + arg->nr_acl_entries); + vmperm->flags |= MEM_BUF_WRAPPER_FLAG_LENDSHARE; + vmperm->memparcel_hdl = arg->memparcel_hdl; + + mutex_unlock(&vmperm->lock); + return 0; + +err_assign: +err_resize: + mutex_unlock(&vmperm->lock); + return ret; +} + +/* + * Kernel API for Sharing, Lending, Receiving or Reclaiming + * a dma-buf from a remote Virtual Machine. + */ +int mem_buf_lend(struct dma_buf *dmabuf, + struct mem_buf_lend_kernel_arg *arg) +{ + return mem_buf_lend_internal(dmabuf, arg, GH_RM_TRANS_TYPE_LEND); +} +EXPORT_SYMBOL(mem_buf_lend); + +int mem_buf_share(struct dma_buf *dmabuf, + struct mem_buf_lend_kernel_arg *arg) +{ + int i, ret, len, found = false; + int *orig_vmids, *vmids = NULL; + int *orig_perms, *perms = NULL; + + len = arg->nr_acl_entries; + for (i = 0; i < len; i++) { + if (arg->vmids[i] == current_vmid) { + found = true; + break; + } + } + + if (found) + return mem_buf_lend_internal(dmabuf, arg, GH_RM_TRANS_TYPE_SHARE); + + vmids = kmalloc_array(len + 1, sizeof(*vmids), GFP_KERNEL); + if (!vmids) + return -ENOMEM; + + perms = kmalloc_array(len + 1, sizeof(*perms), GFP_KERNEL); + if (!perms) { + kfree(vmids); + return -ENOMEM; + } + + /* Add current vmid with RWX permissions to the list */ + memcpy(vmids, arg->vmids, sizeof(*vmids) * len); + memcpy(perms, arg->perms, sizeof(*perms) * len); + vmids[len] = current_vmid; + perms[len] = PERM_READ | PERM_WRITE | PERM_EXEC; + + /* Temporarily switch out the old arrays */ + orig_vmids = arg->vmids; + orig_perms = arg->perms; + arg->vmids = vmids; + arg->perms = perms; + arg->nr_acl_entries += 1; + + ret = mem_buf_lend_internal(dmabuf, arg, GH_RM_TRANS_TYPE_SHARE); + /* Swap back */ + arg->vmids = orig_vmids; + arg->perms = orig_perms; + arg->nr_acl_entries -= 1; + + kfree(vmids); + kfree(perms); + return ret; +} +EXPORT_SYMBOL(mem_buf_share); + +int mem_buf_reclaim(struct dma_buf *dmabuf) +{ + struct mem_buf_vmperm *vmperm; + int ret; + + vmperm = to_mem_buf_vmperm(dmabuf); + if (IS_ERR(vmperm)) { + pr_err_ratelimited("dmabuf ops %ps are not a mem_buf_dma_buf_ops\n", + dmabuf->ops); + return -EINVAL; + } + + mutex_lock(&vmperm->lock); + if (vmperm->flags & MEM_BUF_WRAPPER_FLAG_STATIC_VM) { + pr_err_ratelimited("dma-buf is staticvm type!\n"); + mutex_unlock(&vmperm->lock); + return -EINVAL; + } + + if (!(vmperm->flags & MEM_BUF_WRAPPER_FLAG_LENDSHARE)) { + pr_err_ratelimited("dma-buf isn't lent or shared!\n"); + mutex_unlock(&vmperm->lock); + return -EINVAL; + } + + if (vmperm->flags & MEM_BUF_WRAPPER_FLAG_ACCEPT) { + pr_err_ratelimited("dma-buf not owned by current vm!\n"); + mutex_unlock(&vmperm->lock); + return -EINVAL; + } + + ret = __mem_buf_vmperm_reclaim(vmperm); + mutex_unlock(&vmperm->lock); + return ret; +} +EXPORT_SYMBOL(mem_buf_reclaim); + +bool mem_buf_dma_buf_exclusive_owner(struct dma_buf *dmabuf) +{ + struct mem_buf_vmperm *vmperm; + bool ret = false; + + vmperm = to_mem_buf_vmperm(dmabuf); + if (WARN_ON(IS_ERR(vmperm))) + return false; + + mutex_lock(&vmperm->lock); + ret = !vmperm->flags; + mutex_unlock(&vmperm->lock); + return ret; +} +EXPORT_SYMBOL(mem_buf_dma_buf_exclusive_owner); + +int mem_buf_dma_buf_copy_vmperm(struct dma_buf *dmabuf, int **vmids, + int **perms, int *nr_acl_entries) +{ + struct mem_buf_vmperm *vmperm; + size_t size; + int *vmids_copy, *perms_copy; + int ret; + + vmperm = to_mem_buf_vmperm(dmabuf); + if (IS_ERR(vmperm)) + return PTR_ERR(vmperm); + + mutex_lock(&vmperm->lock); + size = sizeof(*vmids_copy) * vmperm->nr_acl_entries; + vmids_copy = kmemdup(vmperm->vmids, size, GFP_KERNEL); + if (!vmids_copy) { + ret = -ENOMEM; + goto err_vmids; + } + + perms_copy = kmemdup(vmperm->perms, size, GFP_KERNEL); + if (!perms_copy) { + ret = -ENOMEM; + goto err_perms; + } + + *vmids = vmids_copy; + *perms = perms_copy; + *nr_acl_entries = vmperm->nr_acl_entries; + + mutex_unlock(&vmperm->lock); + return 0; + +err_perms: + kfree(vmids_copy); +err_vmids: + mutex_unlock(&vmperm->lock); + return ret; +} +EXPORT_SYMBOL(mem_buf_dma_buf_copy_vmperm); + diff --git a/drivers/soc/qcom/mem_buf/trace-mem-buf.h b/drivers/soc/qcom/mem_buf/trace-mem-buf.h new file mode 100644 index 000000000000..8f7d0c2f6c10 --- /dev/null +++ b/drivers/soc/qcom/mem_buf/trace-mem-buf.h @@ -0,0 +1,299 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020-2021 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM mem_buf + +#if !defined(_TRACE_MEM_BUF_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_MEM_BUF_H +#include +#include +#include + +#include "mem-buf-msgq.h" + +#ifdef CREATE_TRACE_POINTS +static void __maybe_unused gh_acl_to_vmid_perms(struct gh_acl_desc *acl_desc, + u16 *vmids, u8 *perms) +{ + unsigned int i; + + for (i = 0; i < acl_desc->n_acl_entries; i++) { + vmids[i] = acl_desc->acl_entries[i].vmid; + perms[i] = acl_desc->acl_entries[i].perms; + } +} + +static void __maybe_unused +gh_sgl_to_ipa_bases_sizes(struct gh_sgl_desc *sgl_desc, + u64 *ipa_bases, u64 *sizes) +{ + unsigned int i; + + for (i = 0; i < sgl_desc->n_sgl_entries; i++) { + ipa_bases[i] = sgl_desc->sgl_entries[i].ipa_base; + sizes[i] = sgl_desc->sgl_entries[i].size; + } +} + +static char __maybe_unused *mem_type_to_str(enum mem_buf_mem_type type) +{ + if (type == MEM_BUF_ION_MEM_TYPE) + return "ION_MEM_TYPE"; + + return NULL; +} + +static char __maybe_unused *msg_type_to_str(enum mem_buf_msg_type type) +{ + if (type == MEM_BUF_ALLOC_REQ) + return "MEM_BUF_ALLOC_REQ"; + else if (type == MEM_BUF_ALLOC_RESP) + return "MEM_BUF_ALLOC_RESP"; + else if (type == MEM_BUF_ALLOC_RELINQUISH) + return "MEM_BUF_ALLOC_RELINQUISH"; + + return NULL; +} +#endif /* CREATE_TRACE_POINTS */ + +TRACE_EVENT(mem_buf_alloc_info, + + TP_PROTO(size_t size, enum mem_buf_mem_type src_mem_type, + enum mem_buf_mem_type dst_mem_type, + struct gh_acl_desc *acl_desc), + + TP_ARGS(size, src_mem_type, dst_mem_type, acl_desc), + + TP_STRUCT__entry( + __field(size_t, size) + __field(u32, nr_acl_entries) + __string(src_type, mem_type_to_str(src_mem_type)) + __string(dst_type, mem_type_to_str(dst_mem_type)) + __dynamic_array(u16, vmids, acl_desc->n_acl_entries) + __dynamic_array(u8, perms, acl_desc->n_acl_entries) + ), + + TP_fast_assign( + __entry->size = size; + __assign_str(src_type, mem_type_to_str(src_mem_type)); + __assign_str(dst_type, mem_type_to_str(dst_mem_type)); + __entry->nr_acl_entries = acl_desc->n_acl_entries; + gh_acl_to_vmid_perms(acl_desc, __get_dynamic_array(vmids), + __get_dynamic_array(perms)); + ), + + TP_printk("size: 0x%lx src mem type: %s dst mem type: %s nr ACL entries: %d ACL VMIDs: %s ACL Perms: %s", + __entry->size, __get_str(src_type), __get_str(dst_type), + __entry->nr_acl_entries, + __print_array(__get_dynamic_array(vmids), + __entry->nr_acl_entries, sizeof(u16)), + __print_array(__get_dynamic_array(perms), + __entry->nr_acl_entries, sizeof(u8)) + ) +); + +DECLARE_EVENT_CLASS(alloc_req_msg_class, + + TP_PROTO(struct mem_buf_alloc_req *req), + + TP_ARGS(req), + + TP_STRUCT__entry( + __field(u32, txn_id) + __string(msg_type, msg_type_to_str(req->hdr.msg_type)) + __field(u64, size) + __string(src_type, mem_type_to_str(req->src_mem_type)) + __field(u32, nr_acl_entries) + __dynamic_array(u16, vmids, req->acl_desc.n_acl_entries) + __dynamic_array(u8, perms, req->acl_desc.n_acl_entries) + ), + + TP_fast_assign( + __entry->txn_id = req->hdr.txn_id; + __assign_str(msg_type, msg_type_to_str(req->hdr.msg_type)); + __entry->size = req->size; + __assign_str(src_type, mem_type_to_str(req->src_mem_type)); + __entry->nr_acl_entries = req->acl_desc.n_acl_entries; + gh_acl_to_vmid_perms(&req->acl_desc, __get_dynamic_array(vmids), + __get_dynamic_array(perms)); + ), + + TP_printk("txn_id: %d msg_type: %s alloc_sz: 0x%lx src_mem_type: %s nr ACL entries: %d ACL VMIDs: %s ACL Perms: %s", + __entry->txn_id, __get_str(msg_type), __entry->size, + __get_str(src_type), __entry->nr_acl_entries, + __print_array(__get_dynamic_array(vmids), + __entry->nr_acl_entries, sizeof(u16)), + __print_array(__get_dynamic_array(perms), + __entry->nr_acl_entries, sizeof(u8)) + ) +); + +DEFINE_EVENT(alloc_req_msg_class, send_alloc_req, + + TP_PROTO(struct mem_buf_alloc_req *req), + + TP_ARGS(req) +); + +DEFINE_EVENT(alloc_req_msg_class, receive_alloc_req, + + TP_PROTO(struct mem_buf_alloc_req *req), + + TP_ARGS(req) +); + +DECLARE_EVENT_CLASS(relinquish_req_msg_class, + + TP_PROTO(struct mem_buf_alloc_relinquish *rel_req), + + TP_ARGS(rel_req), + + TP_STRUCT__entry( + __string(msg_type, msg_type_to_str(rel_req->hdr.msg_type)) + __field(gh_memparcel_handle_t, hdl) + __field(u32, txn_id) + ), + + TP_fast_assign( + __assign_str(msg_type, msg_type_to_str(rel_req->hdr.msg_type)); + __entry->hdl = rel_req->hdl; + __entry->txn_id = rel_req->hdr.txn_id; + ), + + TP_printk("msg_type: %s memparcel_hdl: 0x%x txn_id: 0x%x", + __get_str(msg_type), __entry->hdl, __entry->txn_id) +); + +DEFINE_EVENT(relinquish_req_msg_class, send_relinquish_msg, + + TP_PROTO(struct mem_buf_alloc_relinquish *rel_req), + + TP_ARGS(rel_req) +); + + +DEFINE_EVENT(relinquish_req_msg_class, receive_relinquish_msg, + + TP_PROTO(struct mem_buf_alloc_relinquish *rel_req), + + TP_ARGS(rel_req) +); + +DECLARE_EVENT_CLASS(alloc_resp_class, + + TP_PROTO(struct mem_buf_alloc_resp *resp), + + TP_ARGS(resp), + + TP_STRUCT__entry( + __field(u32, txn_id) + __string(msg_type, msg_type_to_str(resp->hdr.msg_type)) + __field(s32, ret) + __field(gh_memparcel_handle_t, hdl) + ), + + TP_fast_assign( + __entry->txn_id = resp->hdr.txn_id; + __assign_str(msg_type, msg_type_to_str(resp->hdr.msg_type)); + __entry->ret = resp->ret; + __entry->hdl = resp->hdl; + ), + + TP_printk("txn_id: %d msg_type: %s ret: %d memparcel_hdl: 0x%x", + __entry->txn_id, __get_str(msg_type), __entry->ret, + __entry->hdl + ) +); + +DEFINE_EVENT(alloc_resp_class, send_alloc_resp_msg, + + TP_PROTO(struct mem_buf_alloc_resp *resp), + + TP_ARGS(resp) +); + +DEFINE_EVENT(alloc_resp_class, receive_alloc_resp_msg, + + TP_PROTO(struct mem_buf_alloc_resp *resp), + + TP_ARGS(resp) +); + +TRACE_EVENT(lookup_sgl, + + TP_PROTO(struct gh_sgl_desc *sgl_desc, int ret, + gh_memparcel_handle_t hdl), + + TP_ARGS(sgl_desc, ret, hdl), + + TP_STRUCT__entry( + __field(u16, nr_sgl_entries) + __dynamic_array(u64, ipa_bases, sgl_desc->n_sgl_entries) + __dynamic_array(u64, sizes, sgl_desc->n_sgl_entries) + __field(int, ret) + __field(gh_memparcel_handle_t, hdl) + ), + + TP_fast_assign( + __entry->nr_sgl_entries = sgl_desc->n_sgl_entries; + gh_sgl_to_ipa_bases_sizes(sgl_desc, + __get_dynamic_array(ipa_bases), + __get_dynamic_array(sizes)); + __entry->ret = ret; + __entry->hdl = hdl; + ), + + TP_printk("SGL entries: %d SGL IPA bases: %s SGL sizes: %s ret: %d memparcel_hdl: 0x%x", + __entry->nr_sgl_entries, + __print_array(__get_dynamic_array(ipa_bases), + __entry->nr_sgl_entries, sizeof(u64)), + __print_array(__get_dynamic_array(sizes), + __entry->nr_sgl_entries, sizeof(u64)), + __entry->ret, __entry->hdl + ) +); + +TRACE_EVENT(map_mem_s2, + + TP_PROTO(gh_memparcel_handle_t hdl, struct gh_sgl_desc *sgl_desc), + + TP_ARGS(hdl, sgl_desc), + + TP_STRUCT__entry( + __field(gh_memparcel_handle_t, hdl) + __field(u16, nr_sgl_entries) + __dynamic_array(u64, ipa_bases, sgl_desc->n_sgl_entries) + __dynamic_array(u64, sizes, sgl_desc->n_sgl_entries) + ), + + TP_fast_assign( + __entry->hdl = hdl; + __entry->nr_sgl_entries = sgl_desc->n_sgl_entries; + gh_sgl_to_ipa_bases_sizes(sgl_desc, + __get_dynamic_array(ipa_bases), + __get_dynamic_array(sizes)); + ), + + TP_printk("MEM_ACCEPT successful memparcel hdl: 0x%x SGL entries: %d SGL IPA bases: %s SGL sizes: %s", + __entry->hdl, __entry->nr_sgl_entries, + __print_array(__get_dynamic_array(ipa_bases), + __entry->nr_sgl_entries, sizeof(u64)), + __print_array(__get_dynamic_array(sizes), + __entry->nr_sgl_entries, sizeof(u64)) + ) +); + +#endif /* _TRACE_MEM_BUF_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH ../../../drivers/soc/qcom/mem_buf/ + +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace-mem-buf + +/* This part must be outside protection */ +#include From ab03f053aac5669779ea36cbfcae7d4bc3c867e8 Mon Sep 17 00:00:00 2001 From: Chris Goldsworthy Date: Thu, 16 Jun 2022 14:39:27 -0700 Subject: [PATCH 2/2] mem-buf: Import the DMA-BUF symbol namespace commit 16b0314aa746 ("dma-buf: move dma-buf symbols into the DMA_BUF module namespace") requires us to import the DMA-BUF heap name space into our module, so do this. Change-Id: I8e3f77edb738b86bce96d350475b19b12f9af733 Signed-off-by: Chris Goldsworthy --- drivers/soc/qcom/mem_buf/mem-buf-dev.c | 1 + drivers/soc/qcom/mem_buf/mem-buf.c | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/soc/qcom/mem_buf/mem-buf-dev.c b/drivers/soc/qcom/mem_buf/mem-buf-dev.c index c32c00fbcdb3..4115ac8daa38 100644 --- a/drivers/soc/qcom/mem_buf/mem-buf-dev.c +++ b/drivers/soc/qcom/mem_buf/mem-buf-dev.c @@ -168,3 +168,4 @@ module_exit(mem_buf_dev_exit); MODULE_DESCRIPTION("Qualcomm Technologies, Inc. Memory Buffer Sharing driver"); MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS(DMA_BUF); diff --git a/drivers/soc/qcom/mem_buf/mem-buf.c b/drivers/soc/qcom/mem_buf/mem-buf.c index 42dca0451be9..64b1f3451791 100644 --- a/drivers/soc/qcom/mem_buf/mem-buf.c +++ b/drivers/soc/qcom/mem_buf/mem-buf.c @@ -390,3 +390,4 @@ module_exit(mem_buf_exit); MODULE_DESCRIPTION("Qualcomm Technologies, Inc. Memory Buffer Sharing driver"); MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS(DMA_BUF);