diff --git a/drivers/vfio/pci/Kconfig b/drivers/vfio/pci/Kconfig index 1e82b44bda1a..296bf01e185e 100644 --- a/drivers/vfio/pci/Kconfig +++ b/drivers/vfio/pci/Kconfig @@ -60,6 +60,8 @@ config VFIO_PCI_DMABUF source "drivers/vfio/pci/mlx5/Kconfig" +source "drivers/vfio/pci/ism/Kconfig" + source "drivers/vfio/pci/hisilicon/Kconfig" source "drivers/vfio/pci/pds/Kconfig" diff --git a/drivers/vfio/pci/Makefile b/drivers/vfio/pci/Makefile index e0a0757dd1d2..6138f1bf241d 100644 --- a/drivers/vfio/pci/Makefile +++ b/drivers/vfio/pci/Makefile @@ -11,6 +11,8 @@ obj-$(CONFIG_VFIO_PCI) += vfio-pci.o obj-$(CONFIG_MLX5_VFIO_PCI) += mlx5/ +obj-$(CONFIG_ISM_VFIO_PCI) += ism/ + obj-$(CONFIG_HISI_ACC_VFIO_PCI) += hisilicon/ obj-$(CONFIG_PDS_VFIO_PCI) += pds/ diff --git a/drivers/vfio/pci/ism/Kconfig b/drivers/vfio/pci/ism/Kconfig new file mode 100644 index 000000000000..02f47d25fed2 --- /dev/null +++ b/drivers/vfio/pci/ism/Kconfig @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0 +config ISM_VFIO_PCI + tristate "VFIO support for ISM devices" + depends on S390 + select VFIO_PCI_CORE + help + This provides user space support for IBM Internal Shared Memory (ISM) + Adapter devices using the VFIO framework. + + If you don't know what to do here, say N. diff --git a/drivers/vfio/pci/ism/Makefile b/drivers/vfio/pci/ism/Makefile new file mode 100644 index 000000000000..32cc3c66dd11 --- /dev/null +++ b/drivers/vfio/pci/ism/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_ISM_VFIO_PCI) += ism-vfio-pci.o +ism-vfio-pci-y := main.o diff --git a/drivers/vfio/pci/ism/main.c b/drivers/vfio/pci/ism/main.c new file mode 100644 index 000000000000..f83e09b915ed --- /dev/null +++ b/drivers/vfio/pci/ism/main.c @@ -0,0 +1,408 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * vfio-ISM driver for s390 + * + * Copyright IBM Corp. + */ + +#include +#include "../vfio_pci_priv.h" + +#define ISM_VFIO_PCI_OFFSET_SHIFT 48 +#define ISM_VFIO_PCI_OFFSET_TO_INDEX(off) ((off) >> ISM_VFIO_PCI_OFFSET_SHIFT) +#define ISM_VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << ISM_VFIO_PCI_OFFSET_SHIFT) +#define ISM_VFIO_PCI_OFFSET_MASK (((u64)(1) << ISM_VFIO_PCI_OFFSET_SHIFT) - 1) + +/* + * Use __zpci_load() to bypass automatic use of + * PCI MIO instructions which are not supported on ISM devices + */ +#define ISM_READ(size) \ + static int ism_read##size(struct zpci_dev *zdev, int bar, \ + size_t *filled, char __user *buf, \ + loff_t off) \ + { \ + u64 req, tmp; \ + u##size val; \ + int ret; \ + \ + req = ZPCI_CREATE_REQ(READ_ONCE(zdev->fh), bar, sizeof(val)); \ + ret = __zpci_load(&tmp, req, off); \ + if (ret) \ + return ret; \ + val = (u##size)tmp; \ + if (copy_to_user(buf, &val, sizeof(val))) \ + return -EFAULT; \ + *filled = sizeof(val); \ + return 0; \ + } + +ISM_READ(64); +ISM_READ(32); +ISM_READ(16); +ISM_READ(8); + +struct ism_vfio_pci_core_device { + struct vfio_pci_core_device core_device; + struct kmem_cache *store_block_cache; +}; + +static int ism_vfio_pci_open_device(struct vfio_device *core_vdev) +{ + struct ism_vfio_pci_core_device *ivpcd; + struct vfio_pci_core_device *vdev; + int ret; + + ivpcd = container_of(core_vdev, struct ism_vfio_pci_core_device, + core_device.vdev); + vdev = &ivpcd->core_device; + + ret = vfio_pci_core_enable(vdev); + if (ret) + return ret; + + vfio_pci_core_finish_enable(vdev); + return 0; +} + +/* + * ism_vfio_pci_do_io_r() + * + * On s390, kernel primitives such as ioread() and iowrite() are switched over + * from function-handle-based PCI load/stores instructions to PCI memory-I/O (MIO) + * loads/stores when these are available and not explicitly disabled. Since these + * instructions cannot be used with ISM devices, ensure that classic + * function-handle-based PCI instructions are used instead. + */ +static ssize_t ism_vfio_pci_do_io_r(struct vfio_pci_core_device *vdev, + char __user *buf, loff_t off, size_t count, + int bar) +{ + struct zpci_dev *zdev = to_zpci(vdev->pdev); + ssize_t done = 0; + int ret; + + while (count) { + size_t filled; + + if (count >= 8 && IS_ALIGNED(off, 8)) { + ret = ism_read64(zdev, bar, &filled, buf, off); + if (ret) + return ret; + } else if (count >= 4 && IS_ALIGNED(off, 4)) { + ret = ism_read32(zdev, bar, &filled, buf, off); + if (ret) + return ret; + } else if (count >= 2 && IS_ALIGNED(off, 2)) { + ret = ism_read16(zdev, bar, &filled, buf, off); + if (ret) + return ret; + } else { + ret = ism_read8(zdev, bar, &filled, buf, off); + if (ret) + return ret; + } + + count -= filled; + done += filled; + off += filled; + buf += filled; + } + + return done; +} + +/* + * ism_vfio_pci_do_io_w() + * + * Ensure that the PCI store block (PCISTB) instruction is used as required by the + * ISM device. The ISM device also uses a 256 TiB BAR 0 for write operations, + * which requires a 48bit region address space (ISM_VFIO_PCI_OFFSET_SHIFT). + */ +static ssize_t ism_vfio_pci_do_io_w(struct vfio_pci_core_device *vdev, + char __user *buf, loff_t off, size_t count, + int bar) +{ + struct zpci_dev *zdev = to_zpci(vdev->pdev); + struct ism_vfio_pci_core_device *ivpcd; + ssize_t ret; + void *data; + u64 req; + + if (count > zdev->maxstbl) + return -EINVAL; + if (((off % PAGE_SIZE) + count) > PAGE_SIZE) + return -EINVAL; + + ivpcd = container_of(vdev, struct ism_vfio_pci_core_device, + core_device); + data = kmem_cache_alloc(ivpcd->store_block_cache, GFP_KERNEL); + if (!data) + return -ENOMEM; + + if (copy_from_user(data, buf, count)) { + ret = -EFAULT; + goto out_free; + } + + req = ZPCI_CREATE_REQ(READ_ONCE(zdev->fh), bar, count); + ret = __zpci_store_block(data, req, off); + if (ret) + goto out_free; + + ret = count; + +out_free: + kmem_cache_free(ivpcd->store_block_cache, data); + return ret; +} + +static ssize_t ism_vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, + char __user *buf, size_t count, loff_t *ppos, + bool iswrite) +{ + int bar = ISM_VFIO_PCI_OFFSET_TO_INDEX(*ppos); + loff_t pos = *ppos & ISM_VFIO_PCI_OFFSET_MASK; + resource_size_t end; + ssize_t done = 0; + + if (pci_resource_start(vdev->pdev, bar)) + end = pci_resource_len(vdev->pdev, bar); + else + return -EINVAL; + + if (pos >= end) + return -EINVAL; + + count = min(count, (size_t)(end - pos)); + + if (iswrite) + done = ism_vfio_pci_do_io_w(vdev, buf, pos, count, bar); + else + done = ism_vfio_pci_do_io_r(vdev, buf, pos, count, bar); + + if (done >= 0) + *ppos += done; + + return done; +} + +static ssize_t ism_vfio_pci_config_rw(struct vfio_pci_core_device *vdev, + char __user *buf, size_t count, + loff_t *ppos, bool iswrite) +{ + loff_t pos = *ppos; + size_t done = 0; + int ret = 0; + + pos &= ISM_VFIO_PCI_OFFSET_MASK; + + while (count) { + /* + * zPCI must not use MIO instructions for config space access, + * so we can use common code path here. + */ + ret = vfio_pci_config_rw_single(vdev, buf, count, &pos, iswrite); + if (ret < 0) + return ret; + + count -= ret; + done += ret; + buf += ret; + pos += ret; + } + + *ppos += done; + + return done; +} + +static ssize_t ism_vfio_pci_rw(struct vfio_device *core_vdev, char __user *buf, + size_t count, loff_t *ppos, bool iswrite) +{ + unsigned int index = ISM_VFIO_PCI_OFFSET_TO_INDEX(*ppos); + struct vfio_pci_core_device *vdev; + int ret; + + vdev = container_of(core_vdev, struct vfio_pci_core_device, vdev); + + if (!count) + return 0; + + switch (index) { + case VFIO_PCI_CONFIG_REGION_INDEX: + ret = ism_vfio_pci_config_rw(vdev, buf, count, ppos, iswrite); + break; + + case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: + ret = ism_vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite); + break; + + default: + return -EINVAL; + } + + return ret; +} + +static ssize_t ism_vfio_pci_read(struct vfio_device *core_vdev, + char __user *buf, size_t count, loff_t *ppos) +{ + return ism_vfio_pci_rw(core_vdev, buf, count, ppos, false); +} + +static ssize_t ism_vfio_pci_write(struct vfio_device *core_vdev, + const char __user *buf, size_t count, + loff_t *ppos) +{ + return ism_vfio_pci_rw(core_vdev, (char __user *)buf, count, ppos, + true); +} + +static int ism_vfio_pci_ioctl_get_region_info(struct vfio_device *core_vdev, + struct vfio_region_info *info, + struct vfio_info_cap *caps) +{ + struct vfio_pci_core_device *vdev = + container_of(core_vdev, struct vfio_pci_core_device, vdev); + struct pci_dev *pdev = vdev->pdev; + + switch (info->index) { + case VFIO_PCI_CONFIG_REGION_INDEX: + info->offset = ISM_VFIO_PCI_INDEX_TO_OFFSET(info->index); + info->size = pdev->cfg_size; + info->flags = VFIO_REGION_INFO_FLAG_READ | + VFIO_REGION_INFO_FLAG_WRITE; + break; + case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: + info->offset = ISM_VFIO_PCI_INDEX_TO_OFFSET(info->index); + info->size = pci_resource_len(pdev, info->index); + if (!info->size) { + info->flags = 0; + break; + } + info->flags = VFIO_REGION_INFO_FLAG_READ | + VFIO_REGION_INFO_FLAG_WRITE; + break; + default: + info->offset = 0; + info->size = 0; + info->flags = 0; + return -EINVAL; + } + return 0; +} + +static int ism_vfio_pci_init_dev(struct vfio_device *core_vdev) +{ + struct zpci_dev *zdev = to_zpci(to_pci_dev(core_vdev->dev)); + struct ism_vfio_pci_core_device *ivpcd; + char cache_name[20]; + int ret; + + ivpcd = container_of(core_vdev, struct ism_vfio_pci_core_device, + core_device.vdev); + + snprintf(cache_name, sizeof(cache_name), "ism_sb_fid_%08x", zdev->fid); + + ivpcd->store_block_cache = + kmem_cache_create(cache_name, zdev->maxstbl, + (&(struct kmem_cache_args){ + .align = PAGE_SIZE, + .useroffset = 0, + .usersize = zdev->maxstbl, + }), + (SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT)); + if (!ivpcd->store_block_cache) + return -ENOMEM; + + ret = vfio_pci_core_init_dev(core_vdev); + if (ret) + kmem_cache_destroy(ivpcd->store_block_cache); + + return ret; +} + +static void ism_vfio_pci_release_dev(struct vfio_device *core_vdev) +{ + struct ism_vfio_pci_core_device *ivpcd = container_of( + core_vdev, struct ism_vfio_pci_core_device, core_device.vdev); + + kmem_cache_destroy(ivpcd->store_block_cache); + vfio_pci_core_release_dev(core_vdev); +} + +static const struct vfio_device_ops ism_pci_ops = { + .name = "ism-vfio-pci", + .init = ism_vfio_pci_init_dev, + .release = ism_vfio_pci_release_dev, + .open_device = ism_vfio_pci_open_device, + .close_device = vfio_pci_core_close_device, + .ioctl = vfio_pci_core_ioctl, + .get_region_info_caps = ism_vfio_pci_ioctl_get_region_info, + .device_feature = vfio_pci_core_ioctl_feature, + .read = ism_vfio_pci_read, + .write = ism_vfio_pci_write, + .request = vfio_pci_core_request, + .match = vfio_pci_core_match, + .match_token_uuid = vfio_pci_core_match_token_uuid, + .bind_iommufd = vfio_iommufd_physical_bind, + .unbind_iommufd = vfio_iommufd_physical_unbind, + .attach_ioas = vfio_iommufd_physical_attach_ioas, + .detach_ioas = vfio_iommufd_physical_detach_ioas, +}; + +static int ism_vfio_pci_probe(struct pci_dev *pdev, + const struct pci_device_id *id) +{ + struct ism_vfio_pci_core_device *ivpcd; + int ret; + + ivpcd = vfio_alloc_device(ism_vfio_pci_core_device, core_device.vdev, + &pdev->dev, &ism_pci_ops); + if (IS_ERR(ivpcd)) + return PTR_ERR(ivpcd); + + dev_set_drvdata(&pdev->dev, &ivpcd->core_device); + + ret = vfio_pci_core_register_device(&ivpcd->core_device); + if (ret) + vfio_put_device(&ivpcd->core_device.vdev); + + return ret; +} + +static void ism_vfio_pci_remove(struct pci_dev *pdev) +{ + struct vfio_pci_core_device *core_device; + struct ism_vfio_pci_core_device *ivpcd; + + core_device = dev_get_drvdata(&pdev->dev); + ivpcd = container_of(core_device, struct ism_vfio_pci_core_device, + core_device); + + vfio_pci_core_unregister_device(&ivpcd->core_device); + vfio_put_device(&ivpcd->core_device.vdev); +} + +static const struct pci_device_id ism_device_table[] = { + { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_IBM, + PCI_DEVICE_ID_IBM_ISM) }, + {} +}; +MODULE_DEVICE_TABLE(pci, ism_device_table); + +static struct pci_driver ism_vfio_pci_driver = { + .name = KBUILD_MODNAME, + .id_table = ism_device_table, + .probe = ism_vfio_pci_probe, + .remove = ism_vfio_pci_remove, + .err_handler = &vfio_pci_core_err_handlers, + .driver_managed_dma = true, +}; + +module_pci_driver(ism_vfio_pci_driver); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("vfio-pci variant driver for the IBM Internal Shared Memory (ISM) device"); +MODULE_AUTHOR("IBM Corporation");