diff --git a/Documentation/ABI/testing/sysfs-uevent b/Documentation/ABI/testing/sysfs-uevent index 0b6227706b35..c15e18c47a0e 100644 --- a/Documentation/ABI/testing/sysfs-uevent +++ b/Documentation/ABI/testing/sysfs-uevent @@ -8,7 +8,7 @@ Description: Recognized extended format is:: - ACTION [UUID [KEY=VALUE ...] + ACTION [UUID [KEY=VALUE ...]] The ACTION is compulsory - it is the name of the uevent action (``add``, ``change``, ``remove``). There is no change diff --git a/Documentation/driver-api/infrastructure.rst b/Documentation/driver-api/infrastructure.rst index 35e36fee4238..ac73e60fb09c 100644 --- a/Documentation/driver-api/infrastructure.rst +++ b/Documentation/driver-api/infrastructure.rst @@ -17,6 +17,10 @@ The Basic Device Driver-Model Structures .. kernel-doc:: include/linux/device/driver.h :identifiers: probe_type device_driver +.. kernel-doc:: include/linux/property.h + +.. kernel-doc:: include/linux/fwnode.h + Device Drivers Base ------------------- diff --git a/MAINTAINERS b/MAINTAINERS index d0857c48a4eb..b08bbe1271f8 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -297,7 +297,6 @@ F: include/linux/acpi.h F: include/linux/fwnode.h F: include/linux/fw_table.h F: lib/fw_table.c -F: rust/helpers/acpi.c F: rust/kernel/acpi.rs F: tools/power/acpi/ @@ -7902,6 +7901,7 @@ F: include/linux/fwnode.h F: include/linux/kobj* F: include/linux/ksysfs.h F: include/linux/property.h +F: include/linux/sys_soc.h F: include/linux/sysfs.h F: kernel/ksysfs.c F: lib/kobj* @@ -24604,12 +24604,15 @@ F: drivers/iio/chemical/sps30_i2c.c F: drivers/iio/chemical/sps30_serial.c SERIAL DEVICE BUS -M: Rob Herring +M: Markus Probst L: linux-serial@vger.kernel.org S: Maintained F: Documentation/devicetree/bindings/serial/serial.yaml F: drivers/tty/serdev/ F: include/linux/serdev.h +F: rust/helpers/serdev.c +F: rust/kernel/serdev.rs +F: samples/rust/rust_driver_serdev.rs SERIAL IR RECEIVER M: Sean Young diff --git a/arch/powerpc/platforms/powermac/low_i2c.c b/arch/powerpc/platforms/powermac/low_i2c.c index da72a30ab865..a175a32a222b 100644 --- a/arch/powerpc/platforms/powermac/low_i2c.c +++ b/arch/powerpc/platforms/powermac/low_i2c.c @@ -1471,7 +1471,7 @@ static int __init pmac_i2c_create_platform_devices(void) if (bus->platform_dev == NULL) return -ENOMEM; bus->platform_dev->dev.platform_data = bus; - bus->platform_dev->dev.of_node = bus->busnode; + platform_device_set_of_node(bus->platform_dev, bus->busnode); platform_device_add(bus->platform_dev); } diff --git a/drivers/acpi/arm64/apmt.c b/drivers/acpi/arm64/apmt.c index 91fcdd289e63..8ad049f81f44 100644 --- a/drivers/acpi/arm64/apmt.c +++ b/drivers/acpi/arm64/apmt.c @@ -101,7 +101,7 @@ static int __init apmt_add_platform_device(struct acpi_apmt_node *node, if (ret) goto dev_put; - pdev->dev.fwnode = fwnode; + platform_device_set_fwnode(pdev, fwnode); ret = platform_device_add(pdev); diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c index 34412cd697d8..1020ac953e42 100644 --- a/drivers/acpi/arm64/iort.c +++ b/drivers/acpi/arm64/iort.c @@ -1992,7 +1992,7 @@ static int __init iort_add_platform_device(struct acpi_iort_node *node, goto dev_put; } - pdev->dev.fwnode = fwnode; + platform_device_set_fwnode(pdev, fwnode); if (ops->dev_dma_configure) ops->dev_dma_configure(&pdev->dev, node); diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c index c1876f145ae4..beb51151a660 100644 --- a/drivers/acpi/bus.c +++ b/drivers/acpi/bus.c @@ -871,9 +871,9 @@ const struct acpi_device *acpi_companion_match(const struct device *dev) * identifiers and a _DSD object with the "compatible" property, use that * property to match against the given list of identifiers. */ -bool acpi_of_match_device(const struct acpi_device *adev, - const struct of_device_id *of_match_table, - const struct of_device_id **of_id) +static bool acpi_of_match_device(const struct acpi_device *adev, + const struct of_device_id *of_match_table, + const struct of_device_id **of_id) { const union acpi_object *of_compatible, *obj; int i, nval; diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig index f7d385cbd3ba..43f20ca95a2a 100644 --- a/drivers/base/Kconfig +++ b/drivers/base/Kconfig @@ -222,15 +222,6 @@ config DMA_SHARED_BUFFER APIs extension; the file's descriptor can then be passed on to other driver. -config DMA_FENCE_TRACE - bool "Enable verbose DMA_FENCE_TRACE messages" - depends on DMA_SHARED_BUFFER - help - Enable the DMA_FENCE_TRACE printks. This will add extra - spam to the console log, but will make it easier to diagnose - lockup related problems for dma-buffers shared across multiple - devices. - config GENERIC_ARCH_TOPOLOGY bool help diff --git a/drivers/base/arch_numa.c b/drivers/base/arch_numa.c index d4e426aa51c5..6476227b772c 100644 --- a/drivers/base/arch_numa.c +++ b/drivers/base/arch_numa.c @@ -228,10 +228,6 @@ static int __init numa_init(int (*init_func)(void)) { int ret; - nodes_clear(numa_nodes_parsed); - nodes_clear(node_possible_map); - nodes_clear(node_online_map); - ret = numa_memblks_init(init_func, /* memblock_force_top_down */ false); if (ret < 0) goto out_free_distance; diff --git a/drivers/base/attribute_container.c b/drivers/base/attribute_container.c index 4ad26b8dd6a5..3c456374247b 100644 --- a/drivers/base/attribute_container.c +++ b/drivers/base/attribute_container.c @@ -425,7 +425,12 @@ attribute_container_add_class_device(struct device *classdev) if (error) return error; - return attribute_container_add_attrs(classdev); + + error = attribute_container_add_attrs(classdev); + if (error) + device_del(classdev); + + return error; } /** diff --git a/drivers/base/core.c b/drivers/base/core.c index 4d026682944f..4c0c373998a1 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -127,7 +127,13 @@ static void __fwnode_link_cycle(struct fwnode_link *link) { pr_debug("%pfwf: cycle: depends on %pfwf\n", link->consumer, link->supplier); + + if (link->flags & FWLINK_FLAG_CYCLE) + return; + link->flags |= FWLINK_FLAG_CYCLE; + pr_info("%pfwf: Fixed dependency cycle(s) with %pfwf\n", + link->consumer, link->supplier); } /** @@ -2206,8 +2212,6 @@ static int fw_devlink_create_devlink(struct device *con, if (__fw_devlink_relax_cycles(link->consumer, sup_handle)) { __fwnode_link_cycle(link); pr_debug("----- cycle: end -----\n"); - pr_info("%pfwf: Fixed dependency cycle(s) with %pfwf\n", - link->consumer, sup_handle); } device_links_write_unlock(); diff --git a/drivers/base/dd.c b/drivers/base/dd.c index 60c005223844..f6525a7ee8c5 100644 --- a/drivers/base/dd.c +++ b/drivers/base/dd.c @@ -790,8 +790,8 @@ static int really_probe_debug(struct device *dev, const struct device_driver *dr * CONFIG_DYNAMIC_DEBUG and we want a simple 'initcall_debug' on the * kernel commandline to print this all the time at the debug level. */ - printk(KERN_DEBUG "probe of %s returned %d after %lld usecs\n", - dev_name(dev), ret, ktime_us_delta(rettime, calltime)); + printk(KERN_DEBUG "probe of %s with driver %s returned %d after %lld usecs\n", + dev_name(dev), drv->name, ret, ktime_us_delta(rettime, calltime)); return ret; } diff --git a/drivers/base/firmware_loader/builtin/Makefile b/drivers/base/firmware_loader/builtin/Makefile index 6c067dedc01e..c0264b15d48e 100644 --- a/drivers/base/firmware_loader/builtin/Makefile +++ b/drivers/base/firmware_loader/builtin/Makefile @@ -16,20 +16,24 @@ ASM_ALIGN = $(if $(CONFIG_64BIT),3,2) PROGBITS = $(if $(CONFIG_ARM),%,@)progbits filechk_fwbin = \ - echo "/* Generated by $(src)/Makefile */" ;\ - echo " .section .rodata" ;\ - echo " .p2align 4" ;\ - echo "_fw_$(FWSTR)_bin:" ;\ - echo " .incbin \"$(fwdir)/$(FWNAME)\"" ;\ - echo "_fw_end:" ;\ - echo " .section .rodata.str,\"aMS\",$(PROGBITS),1" ;\ - echo " .p2align $(ASM_ALIGN)" ;\ - echo "_fw_$(FWSTR)_name:" ;\ - echo " .string \"$(FWNAME)\"" ;\ - echo " .section .builtin_fw,\"a\",$(PROGBITS)" ;\ - echo " .p2align $(ASM_ALIGN)" ;\ - echo " $(ASM_WORD) _fw_$(FWSTR)_name" ;\ - echo " $(ASM_WORD) _fw_$(FWSTR)_bin" ;\ + if [ ! -s "$(fwdir)/$(FWNAME)" ]; then \ + echo "error: empty firmware: $(fwdir)/$(FWNAME)" >&2 ;\ + exit 1 ;\ + fi ;\ + echo "/* Generated by $(src)/Makefile */" ;\ + echo " .section .rodata" ;\ + echo " .p2align 4" ;\ + echo "_fw_$(FWSTR)_bin:" ;\ + echo " .incbin \"$(fwdir)/$(FWNAME)\"" ;\ + echo "_fw_end:" ;\ + echo " .section .rodata.str,\"aMS\",$(PROGBITS),1" ;\ + echo " .p2align $(ASM_ALIGN)" ;\ + echo "_fw_$(FWSTR)_name:" ;\ + echo " .string \"$(FWNAME)\"" ;\ + echo " .section .builtin_fw,\"a\",$(PROGBITS)" ;\ + echo " .p2align $(ASM_ALIGN)" ;\ + echo " $(ASM_WORD) _fw_$(FWSTR)_name" ;\ + echo " $(ASM_WORD) _fw_$(FWSTR)_bin" ;\ echo " $(ASM_WORD) _fw_end - _fw_$(FWSTR)_bin" $(obj)/%.gen.S: FORCE diff --git a/drivers/base/firmware_loader/builtin/main.c b/drivers/base/firmware_loader/builtin/main.c index d36befebb1b9..1dcebe8e7f8e 100644 --- a/drivers/base/firmware_loader/builtin/main.c +++ b/drivers/base/firmware_loader/builtin/main.c @@ -53,6 +53,8 @@ bool firmware_request_builtin(struct firmware *fw, const char *name) for (b_fw = __start_builtin_fw; b_fw != __end_builtin_fw; b_fw++) { if (strcmp(name, b_fw->name) == 0) { + if (b_fw->size == 0) + return false; fw->size = b_fw->size; fw->data = b_fw->data; return true; diff --git a/drivers/base/firmware_loader/fallback.c b/drivers/base/firmware_loader/fallback.c index 3ef0b312ae71..00d6b6e59461 100644 --- a/drivers/base/firmware_loader/fallback.c +++ b/drivers/base/firmware_loader/fallback.c @@ -95,6 +95,16 @@ static int fw_load_sysfs_fallback(struct fw_sysfs *fw_sysfs, long timeout) retval = -EINTR; goto out; } + + /* + * device_add() exposes the loading interface before pending_list is + * linked into pending_fw_head, so fw_state_done() may run first. + */ + if (fw_state_is_done(fw_priv)) { + mutex_unlock(&fw_lock); + goto out; + } + list_add(&fw_priv->pending_list, &pending_fw_head); mutex_unlock(&fw_lock); diff --git a/drivers/base/isa.c b/drivers/base/isa.c index 5887e4211f80..4e9f68080f39 100644 --- a/drivers/base/isa.c +++ b/drivers/base/isa.c @@ -166,14 +166,16 @@ static int __init isa_bus_init(void) int error; error = bus_register(&isa_bus_type); - if (!error) { - isa_bus = root_device_register("isa"); - if (IS_ERR(isa_bus)) { - error = PTR_ERR(isa_bus); - bus_unregister(&isa_bus_type); - } + if (error) + return error; + + isa_bus = root_device_register("isa"); + if (IS_ERR(isa_bus)) { + bus_unregister(&isa_bus_type); + return PTR_ERR(isa_bus); } - return error; + + return 0; } postcore_initcall(isa_bus_init); diff --git a/drivers/base/module.c b/drivers/base/module.c index 218aaa096455..6789dca485ab 100644 --- a/drivers/base/module.c +++ b/drivers/base/module.c @@ -42,7 +42,7 @@ int module_add_driver(struct module *mod, const struct device_driver *drv) if (mod) mk = &mod->mkobj; else if (drv->mod_name) { - /* Lookup or create built-in module entry in /sys/modules */ + /* Lookup or create built-in module entry in /sys/module */ mk = lookup_or_create_module_kobject(drv->mod_name); if (mk) { /* remember our module structure */ diff --git a/drivers/base/platform.c b/drivers/base/platform.c index fb9120b0bcfe..66f9ec73d47e 100644 --- a/drivers/base/platform.c +++ b/drivers/base/platform.c @@ -599,19 +599,19 @@ static void platform_device_release(struct device *dev) struct platform_object *pa = container_of(dev, struct platform_object, pdev.dev); - of_node_put(pa->pdev.dev.of_node); + device_remove_software_node(dev); + /* + * If the primary firmware node is a software node, its reference count + * was already decreased by the call to device_remove_software_node(). + */ + if (!is_software_node(dev_fwnode(dev))) + fwnode_handle_put(pa->pdev.dev.fwnode); kfree(pa->pdev.dev.platform_data); kfree(pa->pdev.mfd_cell); kfree(pa->pdev.resource); kfree(pa); } -static void platform_device_release_full(struct device *dev) -{ - device_remove_software_node(dev); - platform_device_release(dev); -} - /** * platform_device_alloc - create a platform device * @name: base name of the device we're adding @@ -619,6 +619,13 @@ static void platform_device_release_full(struct device *dev) * * Create a platform device object which can have other objects attached * to it, and which will have attached objects freed when it is released. + * + * The following fields of the dynamically allocated platform device must not + * be modified manually: resource, num_resources, dev.platform_data, + * dev.of_node and dev.fwnode. Users wishing to do the split platform device + * registration with platform_device_alloc() + platform_device_add() are + * required to use dedicated helpers for adding resources, platform data or + * assigning firmware nodes. */ struct platform_device *platform_device_alloc(const char *name, int id) { @@ -693,6 +700,70 @@ int platform_device_add_data(struct platform_device *pdev, const void *data, } EXPORT_SYMBOL_GPL(platform_device_add_data); +/** + * platform_device_set_of_node - assign an OF node to device + * @pdev: platform device to add the node for + * @np: new device node + * + * Assign an OF node to this platform device. Internally keep track of the + * reference count. Devices created with platform_device_alloc() must use this + * function instead of assigning the node manually. This function must not be + * called for a platform device that already has a software node as its primary + * firmware node assigned. + */ +void platform_device_set_of_node(struct platform_device *pdev, + struct device_node *np) +{ + platform_device_set_fwnode(pdev, of_fwnode_handle(np)); +} +EXPORT_SYMBOL_GPL(platform_device_set_of_node); + +/** + * platform_device_set_fwnode - assign a firmware node to device + * @pdev: platform device to set the node for + * @fwnode: new firmware node + * + * Assign a firmware node to this platform device. Internally keep track of the + * reference count. Devices created with platform_device_alloc() must use this + * function instead of assigning the node manually. This function must not be + * called for a platform device that already has a software node as its primary + * firmware node assigned. + */ +void platform_device_set_fwnode(struct platform_device *pdev, + struct fwnode_handle *fwnode) +{ + /* + * If we call this function for a platform device whose primary + * firmware node is a software node, we'll never end up calling the + * symmetric software_node_notify_remove(). There are no users for this + * right now in the tree so just disallow it. + */ + WARN_ON(is_software_node(dev_fwnode(&pdev->dev))); + fwnode_handle_put(pdev->dev.fwnode); + device_set_node(&pdev->dev, fwnode_handle_get(fwnode)); +} +EXPORT_SYMBOL_GPL(platform_device_set_fwnode); + +/** + * platform_device_set_of_node_from_dev - reuse OF node of another device + * @pdev: platform device to set the node for + * @dev2: device whose OF node to reuse + * + * Reuses the OF node of another device in this platform device while + * internally keeping track of reference counting. This function must not be + * called for a platform device that already has a software node as its primary + * firmware node assigned. + */ +void platform_device_set_of_node_from_dev(struct platform_device *pdev, + const struct device *dev2) +{ + /* See platform_device_set_fwnode(). */ + WARN_ON(is_software_node(dev_fwnode(&pdev->dev))); + device_set_of_node_from_dev(&pdev->dev, dev2); + pdev->dev.fwnode = of_fwnode_handle(pdev->dev.of_node); +} +EXPORT_SYMBOL_GPL(platform_device_set_of_node_from_dev); + /** * platform_device_add - add a platform device to device hierarchy * @pdev: platform device we're adding @@ -868,8 +939,7 @@ struct platform_device *platform_device_register_full(const struct platform_devi return ERR_PTR(-ENOMEM); pdev->dev.parent = pdevinfo->parent; - pdev->dev.fwnode = pdevinfo->fwnode; - pdev->dev.of_node = of_node_get(to_of_node(pdev->dev.fwnode)); + device_set_node(&pdev->dev, fwnode_handle_get(pdevinfo->fwnode)); dev_assign_of_node_reused(&pdev->dev, pdevinfo->of_node_reused); if (pdevinfo->dma_mask) { @@ -890,8 +960,6 @@ struct platform_device *platform_device_register_full(const struct platform_devi ret = device_add_software_node(&pdev->dev, pdevinfo->swnode); if (ret) goto err; - - pdev->dev.release = platform_device_release_full; } else if (pdevinfo->properties) { ret = device_create_managed_software_node(&pdev->dev, pdevinfo->properties, NULL); diff --git a/drivers/base/property.c b/drivers/base/property.c index e08eadd66f4f..b136c339ddae 100644 --- a/drivers/base/property.c +++ b/drivers/base/property.c @@ -7,10 +7,10 @@ * Mika Westerberg */ +#include #include #include #include -#include #include #include #include @@ -517,7 +517,6 @@ EXPORT_SYMBOL_GPL(fwnode_property_read_string); int fwnode_property_match_string(const struct fwnode_handle *fwnode, const char *propname, const char *string) { - const char **values; int nval, ret; nval = fwnode_property_string_array_count(fwnode, propname); @@ -527,20 +526,18 @@ int fwnode_property_match_string(const struct fwnode_handle *fwnode, if (nval == 0) return -ENODATA; - values = kcalloc(nval, sizeof(*values), GFP_KERNEL); + const char **values __free(kfree) = kcalloc(nval, sizeof(*values), GFP_KERNEL); if (!values) return -ENOMEM; ret = fwnode_property_read_string_array(fwnode, propname, values, nval); if (ret < 0) - goto out_free; + return ret; ret = match_string(values, nval, string); if (ret < 0) - ret = -ENODATA; + return -ENODATA; -out_free: - kfree(values); return ret; } EXPORT_SYMBOL_GPL(fwnode_property_match_string); @@ -807,18 +804,31 @@ struct fwnode_handle * fwnode_get_next_child_node(const struct fwnode_handle *fwnode, struct fwnode_handle *child) { + const struct fwnode_handle *parent; + struct fwnode_handle *child_parent __free(fwnode_handle) = NULL; struct fwnode_handle *next; - if (IS_ERR_OR_NULL(fwnode)) + /* + * If this function is in a loop and the previous iteration returned + * an child from fwnode->secondary, then we need to use the secondary + * as parent rather than @fwnode. + */ + if (child) { + child_parent = fwnode_get_parent(child); + parent = child_parent; + } else { + parent = fwnode; + } + if (IS_ERR_OR_NULL(parent)) return NULL; /* Try to find a child in primary fwnode */ - next = fwnode_call_ptr_op(fwnode, get_next_child_node, child); + next = fwnode_call_ptr_op(parent, get_next_child_node, child); if (next) return next; /* When no more children in primary, continue with secondary */ - return fwnode_call_ptr_op(fwnode->secondary, get_next_child_node, child); + return fwnode_get_next_child_node(parent->secondary, NULL); } EXPORT_SYMBOL_GPL(fwnode_get_next_child_node); @@ -1115,8 +1125,9 @@ struct fwnode_handle * fwnode_graph_get_next_endpoint(const struct fwnode_handle *fwnode, struct fwnode_handle *prev) { - struct fwnode_handle *ep, *port_parent = NULL; const struct fwnode_handle *parent; + struct fwnode_handle *port_parent __free(fwnode_handle) = NULL; + struct fwnode_handle *ep; /* * If this function is in a loop and the previous iteration returned @@ -1134,13 +1145,9 @@ fwnode_graph_get_next_endpoint(const struct fwnode_handle *fwnode, ep = fwnode_call_ptr_op(parent, graph_get_next_endpoint, prev); if (ep) - goto out_put_port_parent; + return ep; - ep = fwnode_graph_get_next_endpoint(parent->secondary, NULL); - -out_put_port_parent: - fwnode_handle_put(port_parent); - return ep; + return fwnode_graph_get_next_endpoint(parent->secondary, NULL); } EXPORT_SYMBOL_GPL(fwnode_graph_get_next_endpoint); diff --git a/drivers/base/soc.c b/drivers/base/soc.c index 65ce72d49230..af7d71393774 100644 --- a/drivers/base/soc.c +++ b/drivers/base/soc.c @@ -191,6 +191,7 @@ EXPORT_SYMBOL_GPL(soc_device_unregister); static int __init soc_bus_register(void) { + struct soc_device *soc_dev; int ret; ret = bus_register(&soc_bus_type); @@ -198,10 +199,20 @@ static int __init soc_bus_register(void) return ret; soc_bus_registered = true; - if (early_soc_dev_attr) - return PTR_ERR(soc_device_register(early_soc_dev_attr)); + if (early_soc_dev_attr) { + soc_dev = soc_device_register(early_soc_dev_attr); + if (IS_ERR(soc_dev)) { + ret = PTR_ERR(soc_dev); + goto err_unregister_bus; + } + } return 0; + +err_unregister_bus: + soc_bus_registered = false; + bus_unregister(&soc_bus_type); + return ret; } core_initcall(soc_bus_register); diff --git a/drivers/base/swnode.c b/drivers/base/swnode.c index 1f2315858cc3..84a77ffa3dc4 100644 --- a/drivers/base/swnode.c +++ b/drivers/base/swnode.c @@ -537,7 +537,7 @@ software_node_get_reference_args(const struct fwnode_handle *fwnode, if (prop->is_inline) return -EINVAL; - if ((index + 1) * sizeof(*ref) > prop->length) + if (index >= prop->length / sizeof(*ref)) return -ENOENT; ref_array = prop->pointer; diff --git a/drivers/base/test/Kconfig b/drivers/base/test/Kconfig index 1ecf0791241a..542ce07530a1 100644 --- a/drivers/base/test/Kconfig +++ b/drivers/base/test/Kconfig @@ -17,6 +17,7 @@ config DM_KUNIT_TEST config DRIVER_PE_KUNIT_TEST tristate "KUnit Tests for property entry API" if !KUNIT_ALL_TESTS depends on KUNIT + select OF default KUNIT_ALL_TESTS config DRIVER_SWNODE_KUNIT_TEST diff --git a/drivers/base/test/platform-device-test.c b/drivers/base/test/platform-device-test.c index 6355a2231b74..9ce563f76aad 100644 --- a/drivers/base/test/platform-device-test.c +++ b/drivers/base/test/platform-device-test.c @@ -1,12 +1,15 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include #include #include +#include #include #include +#include #define DEVICE_NAME "test" @@ -253,9 +256,186 @@ static struct kunit_suite platform_device_match_test_suite = { .test_cases = platform_device_match_tests, }; +static int platform_device_swnode_test_probe(struct platform_device *pdev) +{ + return 0; +} + +static struct platform_driver platform_swnode_test_driver = { + .probe = platform_device_swnode_test_probe, + .driver = { + .name = DEVICE_NAME, + }, +}; + +static const struct software_node platform_device_test_swnode = { }; + +/* + * Check that reusing a software node works correctly. If the call to + * platform_device_register_full() fails after adding the secondary firmware + * node, the software node must be unregistered in the device's release() + * callback or the subsequent call to platform_device_register_full() will fail + * with -EBUSY due to the software node already having been registered. + */ +static void platform_device_swnode_add_twice(struct kunit *test) +{ + struct platform_device_info pdevinfo; + struct platform_device *pdev; + struct fwnode_handle *fwnode; + bool bound = false; + int ret; + + fwnode = kunit_kzalloc(test, sizeof(*fwnode), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fwnode); + + ret = kunit_platform_driver_register(test, &platform_swnode_test_driver); + KUNIT_ASSERT_EQ(test, ret, 0); + + fwnode_init(fwnode, NULL); + pdevinfo = (struct platform_device_info){ + .name = DEVICE_NAME, + .id = PLATFORM_DEVID_NONE, + .fwnode = fwnode, + .swnode = &platform_device_test_swnode, + }; + + pdev = platform_device_register_full(&pdevinfo); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pdev); + + wait_for_device_probe(); + scoped_guard(device, &pdev->dev) + bound = device_is_bound(&pdev->dev); + + KUNIT_ASSERT_TRUE(test, bound); + + platform_device_unregister(pdev); + + pdev = platform_device_register_full(&pdevinfo); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pdev); + + wait_for_device_probe(); + scoped_guard(device, &pdev->dev) + bound = device_is_bound(&pdev->dev); + + KUNIT_ASSERT_TRUE(test, bound); + + platform_device_unregister(pdev); +} + +/* + * Check that passing a software node as the primary firmware node of the + * platform device does not result in it being unregistered by the call to + * device_remove_software_node() in its release path. + */ +static void platform_device_swnode_as_primary(struct kunit *test) +{ + struct platform_device_info pdevinfo; + struct platform_device *pdev; + struct fwnode_handle *fwnode; + bool bound = false; + int ret; + + ret = kunit_platform_driver_register(test, &platform_swnode_test_driver); + KUNIT_ASSERT_EQ(test, ret, 0); + + fwnode = kunit_software_node_register(test, &platform_device_test_swnode); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fwnode); + + pdevinfo = (struct platform_device_info){ + .name = DEVICE_NAME, + .id = PLATFORM_DEVID_NONE, + .fwnode = fwnode, + }; + + pdev = platform_device_register_full(&pdevinfo); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pdev); + + wait_for_device_probe(); + scoped_guard(device, &pdev->dev) + bound = device_is_bound(&pdev->dev); + + KUNIT_ASSERT_TRUE(test, bound); + + platform_device_unregister(pdev); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, software_node_fwnode(&platform_device_test_swnode)); +} + +/* + * Check that passing two software nodes to platform_device_register_full() + * fails. + */ +static void platform_device_two_swnodes(struct kunit *test) +{ + static const struct property_entry properties[] = { + PROPERTY_ENTRY_U32("foo", 42), + { } + }; + + struct platform_device_info pdevinfo; + struct platform_device *pdev; + struct fwnode_handle *fwnode; + int ret; + + ret = kunit_platform_driver_register(test, &platform_swnode_test_driver); + KUNIT_ASSERT_EQ(test, ret, 0); + + fwnode = kunit_software_node_register(test, &platform_device_test_swnode); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, fwnode); + + pdevinfo = (struct platform_device_info){ + .name = DEVICE_NAME, + .id = PLATFORM_DEVID_NONE, + .fwnode = fwnode, + .swnode = &platform_device_test_swnode, + }; + + pdev = platform_device_register_full(&pdevinfo); + KUNIT_ASSERT_TRUE(test, IS_ERR(pdev)); + KUNIT_ASSERT_EQ_MSG(test, PTR_ERR(pdev), -EINVAL, + "Expected errno == -EINVAL, got: %pe", pdev); + + pdevinfo = (struct platform_device_info){ + .name = DEVICE_NAME, + .id = PLATFORM_DEVID_NONE, + .swnode = &platform_device_test_swnode, + .properties = properties, + }; + + pdev = platform_device_register_full(&pdevinfo); + KUNIT_ASSERT_TRUE(test, IS_ERR(pdev)); + KUNIT_ASSERT_EQ_MSG(test, PTR_ERR(pdev), -EINVAL, + "Expected errno == -EINVAL, got: %pe", pdev); + + pdevinfo = (struct platform_device_info){ + .name = DEVICE_NAME, + .id = PLATFORM_DEVID_NONE, + .fwnode = fwnode, + .properties = properties, + }; + + pdev = platform_device_register_full(&pdevinfo); + KUNIT_ASSERT_TRUE(test, IS_ERR(pdev)); + KUNIT_ASSERT_EQ_MSG(test, PTR_ERR(pdev), -EINVAL, + "Expected errno == -EINVAL, got: %pe", pdev); +} + +static struct kunit_case platform_device_swnode_tests[] = { + KUNIT_CASE(platform_device_swnode_add_twice), + KUNIT_CASE(platform_device_swnode_as_primary), + KUNIT_CASE(platform_device_two_swnodes), + {} +}; + +static struct kunit_suite platform_device_swnode_test_suite = { + .name = "platform-device-swnode", + .test_cases = platform_device_swnode_tests, +}; + kunit_test_suites( &platform_device_devm_test_suite, &platform_device_match_test_suite, + &platform_device_swnode_test_suite, ); MODULE_DESCRIPTION("Test module for platform devices"); diff --git a/drivers/base/test/property-entry-test.c b/drivers/base/test/property-entry-test.c index a8657eb06f94..855e73b9b21f 100644 --- a/drivers/base/test/property-entry-test.c +++ b/drivers/base/test/property-entry-test.c @@ -4,6 +4,8 @@ // Copyright 2019 Google LLC. #include + +#include #include #include @@ -489,6 +491,139 @@ static void pe_test_reference(struct kunit *test) software_node_unregister_node_group(group); } +static struct fwnode_handle *create_device_node(struct kunit *test, + const char *name, + const char *full_name, + struct device_node *parent) +{ + struct device_node *node; + + node = kunit_kzalloc(test, sizeof(*node), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, node); + + node->name = kunit_kstrdup(test, name, GFP_KERNEL); + node->full_name = kunit_kstrdup(test, full_name, GFP_KERNEL); + + if (parent) { + node->sibling = parent->child; + /* set the node as the first child of the parent */ + parent->child = node; + node->parent = parent; + } + + of_node_init(node); + return of_fwnode_handle(node); +} + +/* Verifies that fwnode_for_each_child_node() can output correct children */ +static void pe_test_child_iteration(struct kunit *test) +{ + struct fwnode_handle *of_node, *of_node1; + struct fwnode_handle *sw_node, *sw_node1; + struct fwnode_handle *child; + int error, i, num; + + static const struct software_node node = { .name = "sw" }; + static const struct software_node node1 = { .name = "sw-1", .parent = &node}; + static const struct software_node node2 = { .name = "sw-2", .parent = &node}; + static const struct software_node node3 = { .name = "sw-3", .parent = &node}; + static const struct software_node *group[] = { &node, &node1, &node2, &node3, NULL }; + + static const char * const of_child_array[] = { "of-1", "of-2", "of-3" }; + static const char * const sw_child_array[] = { "sw-1", "sw-2", "sw-3" }; + static const char * const of_sw_child_array[] = { "of-1", "of-2", "of-3", + "sw-1", "sw-2", "sw-3" }; + static const char * const sw_of_child_array[] = { "sw-1", "sw-2", "sw-3", + "of-1", "of-2", "of-3" }; + + /* 1. Test OF node child iteration */ + + of_node = create_device_node(test, "of", "of", NULL); + create_device_node(test, "of", "of-3", to_of_node(of_node)); + create_device_node(test, "of", "of-2", to_of_node(of_node)); + of_node1 = create_device_node(test, "of", "of-1", to_of_node(of_node)); + + i = 0; + num = ARRAY_SIZE(of_child_array); + fwnode_for_each_child_node(of_node, child) { + KUNIT_ASSERT_LT(test, i, num); + KUNIT_EXPECT_STREQ(test, of_child_array[i++], fwnode_get_name(child)); + } + KUNIT_EXPECT_PTR_EQ(test, child, NULL); + + /* 2. Test SW node child iteration */ + + error = software_node_register_node_group(group); + KUNIT_ASSERT_EQ(test, error, 0); + + sw_node = software_node_fwnode(&node); + + i = 0; + num = ARRAY_SIZE(sw_child_array); + fwnode_for_each_child_node(sw_node, child) { + KUNIT_ASSERT_LT(test, i, num); + KUNIT_EXPECT_STREQ(test, sw_child_array[i++], fwnode_get_name(child)); + } + KUNIT_EXPECT_PTR_EQ(test, child, NULL); + + /* 3. Test OF (primary) + SW (secondary) node child iteration */ + + of_node->secondary = sw_node; + sw_node->secondary = ERR_PTR(-ENODEV); + + i = 0; + num = ARRAY_SIZE(of_sw_child_array); + fwnode_for_each_child_node(of_node, child) { + KUNIT_ASSERT_LT(test, i, num); + KUNIT_EXPECT_STREQ(test, of_sw_child_array[i++], fwnode_get_name(child)); + } + KUNIT_EXPECT_PTR_EQ(test, child, NULL); + + /* 4. Test SW (primary) + OF (secondary) node child iteration */ + + sw_node->secondary = of_node; + of_node->secondary = ERR_PTR(-ENODEV); + + i = 0; + num = ARRAY_SIZE(sw_of_child_array); + fwnode_for_each_child_node(sw_node, child) { + KUNIT_ASSERT_LT(test, i, num); + KUNIT_EXPECT_STREQ(test, sw_of_child_array[i++], fwnode_get_name(child)); + } + KUNIT_EXPECT_PTR_EQ(test, child, NULL); + + /* 5. Test OF (primary) + SW (secondary, but no children) node child iteration */ + + sw_node1 = software_node_fwnode(&node1); + of_node->secondary = sw_node1; + sw_node->secondary = ERR_PTR(-ENODEV); + + i = 0; + num = ARRAY_SIZE(of_child_array); + fwnode_for_each_child_node(of_node, child) { + KUNIT_ASSERT_LT(test, i, num); + KUNIT_EXPECT_STREQ(test, of_child_array[i++], fwnode_get_name(child)); + } + KUNIT_EXPECT_PTR_EQ(test, child, NULL); + + /* 6. Test SW (primary) + OF (secondary, but no children) node child iteration */ + + sw_node->secondary = of_node1; + of_node->secondary = ERR_PTR(-ENODEV); + + i = 0; + num = ARRAY_SIZE(sw_child_array); + fwnode_for_each_child_node(sw_node, child) { + KUNIT_ASSERT_LT(test, i, num); + KUNIT_EXPECT_STREQ(test, sw_child_array[i++], fwnode_get_name(child)); + } + KUNIT_EXPECT_PTR_EQ(test, child, NULL); + + of_node->secondary = NULL; + sw_node->secondary = NULL; + software_node_unregister_node_group(group); +} + static struct kunit_case property_entry_test_cases[] = { KUNIT_CASE(pe_test_uints), KUNIT_CASE(pe_test_uint_arrays), @@ -497,6 +632,7 @@ static struct kunit_case property_entry_test_cases[] = { KUNIT_CASE(pe_test_move_inline_u8), KUNIT_CASE(pe_test_move_inline_str), KUNIT_CASE(pe_test_reference), + KUNIT_CASE(pe_test_child_iteration), { } }; @@ -509,4 +645,5 @@ kunit_test_suite(property_entry_test_suite); MODULE_DESCRIPTION("Test module for the property entry API"); MODULE_AUTHOR("Dmitry Torokhov "); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); MODULE_LICENSE("GPL"); diff --git a/drivers/base/transport_class.c b/drivers/base/transport_class.c index 416e9f819df5..351c3d3ce6a0 100644 --- a/drivers/base/transport_class.c +++ b/drivers/base/transport_class.c @@ -168,11 +168,14 @@ static int transport_add_class_device(struct attribute_container *cont, if (tcont->encryption) { error = sysfs_create_group(&classdev->kobj, tcont->encryption); if (error) - goto err_del; + goto err_del_statistics; } return 0; +err_del_statistics: + if (tcont->statistics) + sysfs_remove_group(&classdev->kobj, tcont->statistics); err_del: attribute_container_class_device_del(classdev); err_remove: diff --git a/drivers/cpufreq/rcpufreq_dt.rs b/drivers/cpufreq/rcpufreq_dt.rs index 5d27857261da..d7ead60bf8c2 100644 --- a/drivers/cpufreq/rcpufreq_dt.rs +++ b/drivers/cpufreq/rcpufreq_dt.rs @@ -201,7 +201,6 @@ fn register_em(policy: &mut cpufreq::Policy) { kernel::of_device_table!( OF_TABLE, - MODULE_OF_TABLE, ::IdInfo, [(of::DeviceId::new(c"operating-points-v2"), ())] ); diff --git a/drivers/gpu/drm/nova/driver.rs b/drivers/gpu/drm/nova/driver.rs index 48933d86ddda..43f15cdfeb09 100644 --- a/drivers/gpu/drm/nova/driver.rs +++ b/drivers/gpu/drm/nova/driver.rs @@ -43,7 +43,6 @@ pub(crate) struct NovaData { kernel::auxiliary_device_table!( AUX_TABLE, - MODULE_AUX_TABLE, ::IdInfo, [( auxiliary::DeviceId::new(NOVA_CORE_MODULE_NAME, AUXILIARY_NAME), diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index d063bc664cc1..218e9af899c7 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -87,7 +87,6 @@ fn issue_soft_reset(dev: &Device, iomem: &IoMem<'_>) -> Result { kernel::of_device_table!( OF_TABLE, - MODULE_OF_TABLE, ::IdInfo, [ (of::DeviceId::new(c"rockchip,rk3588-mali"), ()), diff --git a/drivers/gpu/drm/xe/xe_i2c.c b/drivers/gpu/drm/xe/xe_i2c.c index f05f23221c1b..c61d5db07309 100644 --- a/drivers/gpu/drm/xe/xe_i2c.c +++ b/drivers/gpu/drm/xe/xe_i2c.c @@ -93,14 +93,9 @@ static int xe_i2c_register_adapter(struct xe_i2c *i2c) { struct pci_dev *pci = to_pci_dev(i2c->drm_dev); struct platform_device *pdev; - struct fwnode_handle *fwnode; int ret; u32 id; - fwnode = fwnode_create_software_node(xe_i2c_adapter_properties, NULL); - if (IS_ERR(fwnode)) - return PTR_ERR(fwnode); - id = (pci_domain_nr(pci->bus) << 16) | pci_dev_id(pci); /* @@ -110,10 +105,14 @@ static int xe_i2c_register_adapter(struct xe_i2c *i2c) * platform_device_register_full() is done. */ pdev = platform_device_alloc(adapter_name, id); - if (!pdev) { - ret = -ENOMEM; - goto err_fwnode_remove; - } + if (!pdev) + return -ENOMEM; + + ret = device_create_managed_software_node(&pdev->dev, + xe_i2c_adapter_properties, + NULL); + if (ret) + goto err_pdev_put; if (i2c->adapter_irq) { struct resource res; @@ -126,8 +125,6 @@ static int xe_i2c_register_adapter(struct xe_i2c *i2c) } pdev->dev.parent = i2c->drm_dev; - pdev->dev.fwnode = fwnode; - i2c->adapter_node = fwnode; i2c->pdev = pdev; ret = platform_device_add(pdev); @@ -138,8 +135,6 @@ static int xe_i2c_register_adapter(struct xe_i2c *i2c) err_pdev_put: platform_device_put(pdev); -err_fwnode_remove: - fwnode_remove_software_node(fwnode); return ret; } @@ -147,7 +142,6 @@ static int xe_i2c_register_adapter(struct xe_i2c *i2c) static void xe_i2c_unregister_adapter(struct xe_i2c *i2c) { platform_device_unregister(i2c->pdev); - fwnode_remove_software_node(i2c->adapter_node); } /** diff --git a/drivers/gpu/drm/xe/xe_i2c.h b/drivers/gpu/drm/xe/xe_i2c.h index 425d8160835f..b28229f056c5 100644 --- a/drivers/gpu/drm/xe/xe_i2c.h +++ b/drivers/gpu/drm/xe/xe_i2c.h @@ -30,7 +30,6 @@ struct xe_i2c_endpoint { }; struct xe_i2c { - struct fwnode_handle *adapter_node; struct platform_device *pdev; struct i2c_adapter *adapter; struct i2c_client *client[XE_I2C_MAX_CLIENTS]; diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index 5738d4ac521b..bbd93959e0b2 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -15,7 +15,7 @@ Atomic, Relaxed, // }, - types::ForLt, + types::CovariantForLt, }; use crate::gpu::Gpu; @@ -29,7 +29,7 @@ pub(crate) struct NovaCore<'bound> { pub(crate) gpu: Gpu<'bound>, bar: pci::Bar<'bound, BAR0_SIZE>, #[allow(clippy::type_complexity)] - _reg: auxiliary::Registration<'bound, ForLt!(())>, + _reg: auxiliary::Registration<'bound, CovariantForLt!(())>, } pub(crate) struct NovaCoreDriver; @@ -40,7 +40,6 @@ pub(crate) struct NovaCore<'bound> { kernel::pci_device_table!( PCI_TABLE, - MODULE_PCI_TABLE, ::IdInfo, [ // Modern NVIDIA GPUs will show up as either VGA or 3D controllers. @@ -70,7 +69,7 @@ impl pci::Driver for NovaCoreDriver { fn probe<'bound>( pdev: &'bound pci::Device>, - _info: &'bound Self::IdInfo, + _info: Option<&'bound Self::IdInfo>, ) -> impl PinInit, Error> + 'bound { pin_init::pin_init_scope(move || { dev_dbg!(pdev, "Probe Nova Core GPU driver.\n"); diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index b3c91731db45..b603b0bd2692 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -285,10 +285,10 @@ pub(crate) struct Gpu<'gpu> { } impl<'gpu> Gpu<'gpu> { - pub(crate) fn new( - pdev: &'gpu pci::Device>, + pub(crate) fn new<'a>( + pdev: &'gpu pci::Device>, bar: Bar0<'gpu>, - ) -> impl PinInit + 'gpu { + ) -> impl PinInit + use<'gpu, 'a> { try_pin_init!(Self { device: pdev.as_ref(), spec: Spec::new(pdev.as_ref(), bar).inspect(|spec| { diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs index 69175ca3315c..cfa7553cd820 100644 --- a/drivers/gpu/nova-core/gsp.rs +++ b/drivers/gpu/nova-core/gsp.rs @@ -9,14 +9,16 @@ dma::{ Coherent, CoherentBox, + CoherentView, DmaAddress, // }, + io::{ + io_project, + io_write, + Io, // + }, pci, - prelude::*, - transmute::{ - AsBytes, - FromBytes, // - }, // + prelude::*, // }; pub(crate) mod cmdq; @@ -48,21 +50,21 @@ /// Array of page table entries, as understood by the GSP bootloader. #[repr(C)] +#[derive(FromBytes, IntoBytes)] struct PteArray([u64; NUM_ENTRIES]); -/// SAFETY: arrays of `u64` implement `FromBytes` and we are but a wrapper around one. -unsafe impl FromBytes for PteArray {} - -/// SAFETY: arrays of `u64` implement `AsBytes` and we are but a wrapper around one. -unsafe impl AsBytes for PteArray {} - impl PteArray { - /// Returns the page table entry for `index`, for a mapping starting at `start`. - // TODO: Replace with `IoView` projection once available. - fn entry(start: DmaAddress, index: usize) -> Result { - start - .checked_add(num::usize_as_u64(index) << GSP_PAGE_SHIFT) - .ok_or(EOVERFLOW) + /// Initialize a new page table array mapping `NUM_PAGES` GSP pages starting at address `start`. + fn init(view: CoherentView<'_, Self>, start: DmaAddress) -> Result<()> { + for i in 0..NUM_PAGES { + io_write!(view, .0[build: i], + start + .checked_add(num::usize_as_u64(i) << GSP_PAGE_SHIFT) + .ok_or(EOVERFLOW)? + ); + } + + Ok(()) } } @@ -89,17 +91,12 @@ fn new(dev: &device::Device) -> Result { let start_addr = obj.0.dma_handle(); - // SAFETY: `obj` has just been created and we are its sole user. - let pte_region = unsafe { - &mut obj.0.as_mut()[size_of::()..][..RM_LOG_BUFFER_NUM_PAGES * size_of::()] - }; - - // Write values one by one to avoid an on-stack instance of `PteArray`. - for (i, chunk) in pte_region.chunks_exact_mut(size_of::()).enumerate() { - let pte_value = PteArray::<0>::entry(start_addr, i)?; - - chunk.copy_from_slice(&pte_value.to_ne_bytes()); - } + let pte_view = io_project!( + obj.0, + [build: size_of::()..][build: ..RM_LOG_BUFFER_NUM_PAGES * size_of::()] + ) + .try_cast::>()?; + PteArray::init(pte_view, start_addr)?; Ok(obj) } diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs index 3c68a66770d3..9349717b982b 100644 --- a/drivers/gpu/nova-core/gsp/cmdq.rs +++ b/drivers/gpu/nova-core/gsp/cmdq.rs @@ -2,16 +2,23 @@ mod continuation; -use core::mem; +use core::{ + mem, + sync::atomic::{ + fence, + Ordering, // + }, +}; use kernel::{ device, dma::{ Coherent, + CoherentBox, DmaAddress, // }, - dma_write, io::{ + io_project, poll::read_poll_timeout, Io, // }, @@ -171,20 +178,18 @@ struct MsgqData { #[repr(C)] // There is no struct defined for this in the open-gpu-kernel-source headers. // Instead it is defined by code in `GspMsgQueuesInit()`. -// TODO: Revert to private once `IoView` projections replace the `gsp_mem` module. -pub(super) struct Msgq { +struct Msgq { /// Header for sending messages, including the write pointer. - pub(super) tx: MsgqTxHeader, + tx: MsgqTxHeader, /// Header for receiving messages, including the read pointer. - pub(super) rx: MsgqRxHeader, + rx: MsgqRxHeader, /// The message queue proper. msgq: MsgqData, } /// Structure shared between the driver and the GSP and containing the command and message queues. #[repr(C)] -// TODO: Revert to private once `IoView` projections replace the `gsp_mem` module. -pub(super) struct GspMem { +struct GspMem { /// Self-mapping page table entries. ptes: PteArray<{ Self::PTE_ARRAY_SIZE }>, /// CPU queue: the driver writes commands here, and the GSP reads them. It also contains the @@ -192,13 +197,13 @@ pub(super) struct GspMem { /// index into the GSP queue. /// /// This member is read-only for the GSP. - pub(super) cpuq: Msgq, + cpuq: Msgq, /// GSP queue: the GSP writes messages here, and the driver reads them. It also contains the /// write and read pointers that the GSP updates. This means that the read pointer here is an /// index into the CPU queue. /// /// This member is read-only for the driver. - pub(super) gspq: Msgq, + gspq: Msgq, } impl GspMem { @@ -232,20 +237,12 @@ fn new(dev: &device::Device) -> Result { const MSGQ_SIZE: u32 = num::usize_into_u32::<{ size_of::() }>(); const RX_HDR_OFF: u32 = num::usize_into_u32::<{ mem::offset_of!(Msgq, rx) }>(); - let gsp_mem = Coherent::::zeroed(dev, GFP_KERNEL)?; + let mut gsp_mem = CoherentBox::::zeroed(dev, GFP_KERNEL)?; + gsp_mem.cpuq.tx = MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES); + gsp_mem.cpuq.rx = MsgqRxHeader::new(); - let start = gsp_mem.dma_handle(); - // Write values one by one to avoid an on-stack instance of `PteArray`. - for i in 0..GspMem::PTE_ARRAY_SIZE { - dma_write!(gsp_mem, .ptes.0[build: i], PteArray::<0>::entry(start, i)?); - } - - dma_write!( - gsp_mem, - .cpuq.tx, - MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES) - ); - dma_write!(gsp_mem, .cpuq.rx, MsgqRxHeader::new()); + let gsp_mem: Coherent<_> = gsp_mem.into(); + PteArray::init(io_project!(gsp_mem, .ptes), gsp_mem.dma_handle())?; Ok(Self(gsp_mem)) } @@ -406,7 +403,7 @@ fn allocate_command(&mut self, size: usize, timeout: Delta) -> Result u32 { - super::fw::gsp_mem::gsp_write_ptr(&self.0) + MsgqTxHeader::write_ptr(io_project!(self.0, .gspq.tx)) % MSGQ_NUM_PAGES } // Returns the index of the memory page the GSP will read the next command from. @@ -415,7 +412,7 @@ fn gsp_write_ptr(&self) -> u32 { // // - The returned value is within `0..MSGQ_NUM_PAGES`. fn gsp_read_ptr(&self) -> u32 { - super::fw::gsp_mem::gsp_read_ptr(&self.0) + MsgqRxHeader::read_ptr(io_project!(self.0, .gspq.rx)) % MSGQ_NUM_PAGES } // Returns the index of the memory page the CPU can read the next message from. @@ -424,12 +421,18 @@ fn gsp_read_ptr(&self) -> u32 { // // - The returned value is within `0..MSGQ_NUM_PAGES`. fn cpu_read_ptr(&self) -> u32 { - super::fw::gsp_mem::cpu_read_ptr(&self.0) + MsgqRxHeader::read_ptr(io_project!(self.0, .cpuq.rx)) % MSGQ_NUM_PAGES } // Informs the GSP that it can send `elem_count` new pages into the message queue. fn advance_cpu_read_ptr(&mut self, elem_count: u32) { - super::fw::gsp_mem::advance_cpu_read_ptr(&self.0, elem_count) + let rx = io_project!(self.0, .cpuq.rx); + let rptr = MsgqRxHeader::read_ptr(rx).wrapping_add(elem_count) % MSGQ_NUM_PAGES; + + // Ensure read pointer is properly ordered. + fence(Ordering::SeqCst); + + MsgqRxHeader::set_read_ptr(rx, rptr) } // Returns the index of the memory page the CPU can write the next command to. @@ -438,12 +441,17 @@ fn advance_cpu_read_ptr(&mut self, elem_count: u32) { // // - The returned value is within `0..MSGQ_NUM_PAGES`. fn cpu_write_ptr(&self) -> u32 { - super::fw::gsp_mem::cpu_write_ptr(&self.0) + MsgqTxHeader::write_ptr(io_project!(self.0, .cpuq.tx)) % MSGQ_NUM_PAGES } // Informs the GSP that it can process `elem_count` new pages from the command queue. fn advance_cpu_write_ptr(&mut self, elem_count: u32) { - super::fw::gsp_mem::advance_cpu_write_ptr(&self.0, elem_count) + let tx = io_project!(self.0, .cpuq.tx); + let wptr = MsgqTxHeader::write_ptr(tx).wrapping_add(elem_count) % MSGQ_NUM_PAGES; + MsgqTxHeader::set_write_ptr(tx, wptr); + + // Ensure all command data is visible before triggering the GSP read. + fence(Ordering::SeqCst); } } diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs index 4db0cfa4dc4d..b0e7de328eaf 100644 --- a/drivers/gpu/nova-core/gsp/fw.rs +++ b/drivers/gpu/nova-core/gsp/fw.rs @@ -10,7 +10,14 @@ use core::ops::Range; use kernel::{ - dma::Coherent, + dma::{ + Coherent, + CoherentView, // + }, + io::{ + io_read, + io_write, // + }, prelude::*, ptr::{ Alignable, @@ -44,59 +51,6 @@ }, }; -// TODO: Replace with `IoView` projections once available. -pub(super) mod gsp_mem { - use core::sync::atomic::{ - fence, - Ordering, // - }; - - use kernel::{ - dma::Coherent, - dma_read, - dma_write, // - }; - - use crate::gsp::cmdq::{ - GspMem, - MSGQ_NUM_PAGES, // - }; - - pub(in crate::gsp) fn gsp_write_ptr(qs: &Coherent) -> u32 { - dma_read!(qs, .gspq.tx.0.writePtr) % MSGQ_NUM_PAGES - } - - pub(in crate::gsp) fn gsp_read_ptr(qs: &Coherent) -> u32 { - dma_read!(qs, .gspq.rx.0.readPtr) % MSGQ_NUM_PAGES - } - - pub(in crate::gsp) fn cpu_read_ptr(qs: &Coherent) -> u32 { - dma_read!(qs, .cpuq.rx.0.readPtr) % MSGQ_NUM_PAGES - } - - pub(in crate::gsp) fn advance_cpu_read_ptr(qs: &Coherent, count: u32) { - let rptr = cpu_read_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES; - - // Ensure read pointer is properly ordered. - fence(Ordering::SeqCst); - - dma_write!(qs, .cpuq.rx.0.readPtr, rptr); - } - - pub(in crate::gsp) fn cpu_write_ptr(qs: &Coherent) -> u32 { - dma_read!(qs, .cpuq.tx.0.writePtr) % MSGQ_NUM_PAGES - } - - pub(in crate::gsp) fn advance_cpu_write_ptr(qs: &Coherent, count: u32) { - let wptr = cpu_write_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES; - - dma_write!(qs, .cpuq.tx.0.writePtr, wptr); - - // Ensure all command data is visible before triggering the GSP read. - fence(Ordering::SeqCst); - } -} - /// Maximum size of a single GSP message queue element in bytes. pub(crate) const GSP_MSG_QUEUE_ELEMENT_SIZE_MAX: usize = num::u32_as_usize(bindings::GSP_MSG_QUEUE_ELEMENT_SIZE_MAX); @@ -720,6 +674,16 @@ pub(crate) fn new(msgq_size: u32, rx_hdr_offset: u32, msg_count: u32) -> Self { entryOff: num::usize_into_u32::(), }) } + + /// Returns the value of the write pointer for this queue. + pub(crate) fn write_ptr(this: CoherentView<'_, Self>) -> u32 { + io_read!(this, .0.writePtr) + } + + /// Sets the value of the write pointer for this queue. + pub(crate) fn set_write_ptr(this: CoherentView<'_, Self>, val: u32) { + io_write!(this, .0.writePtr, val) + } } // SAFETY: Padding is explicit and does not contain uninitialized data. @@ -735,6 +699,16 @@ impl MsgqRxHeader { pub(crate) fn new() -> Self { Self(Default::default()) } + + /// Returns the value of the read pointer for this queue. + pub(crate) fn read_ptr(this: CoherentView<'_, Self>) -> u32 { + io_read!(this, .0.readPtr) + } + + /// Sets the value of the read pointer for this queue. + pub(crate) fn set_read_ptr(this: CoherentView<'_, Self>, val: u32) { + io_write!(this, .0.readPtr, val) + } } // SAFETY: Padding is explicit and does not contain uninitialized data. diff --git a/drivers/i2c/busses/i2c-pxa-pci.c b/drivers/i2c/busses/i2c-pxa-pci.c index dbd542300f80..92a0647f08c6 100644 --- a/drivers/i2c/busses/i2c-pxa-pci.c +++ b/drivers/i2c/busses/i2c-pxa-pci.c @@ -76,7 +76,8 @@ static struct platform_device *add_i2c_device(struct pci_dev *dev, int bar) goto out; } pdev->dev.parent = &dev->dev; - pdev->dev.of_node = child; + + platform_device_set_of_node(pdev, child); ret = platform_device_add_resources(pdev, res, ARRAY_SIZE(res)); if (ret) diff --git a/drivers/iommu/fsl_pamu.c b/drivers/iommu/fsl_pamu.c index 25aa477a95a9..c83bbc3faad5 100644 --- a/drivers/iommu/fsl_pamu.c +++ b/drivers/iommu/fsl_pamu.c @@ -8,6 +8,7 @@ #include "fsl_pamu.h" +#include #include #include #include @@ -933,7 +934,6 @@ static struct platform_driver fsl_of_pamu_driver = { static __init int fsl_pamu_init(void) { struct platform_device *pdev = NULL; - struct device_node *np; int ret; /* @@ -955,7 +955,8 @@ static __init int fsl_pamu_init(void) * PAMU node would require significant changes to a lot of code. */ - np = of_find_compatible_node(NULL, NULL, "fsl,pamu"); + struct device_node *np __free(device_node) = + of_find_compatible_node(NULL, NULL, "fsl,pamu"); if (!np) { pr_err("could not find a PAMU node\n"); return -ENODEV; @@ -964,7 +965,7 @@ static __init int fsl_pamu_init(void) ret = platform_driver_register(&fsl_of_pamu_driver); if (ret) { pr_err("could not register driver (err=%i)\n", ret); - goto error_driver_register; + return ret; } pdev = platform_device_alloc("fsl-of-pamu", 0); @@ -973,7 +974,8 @@ static __init int fsl_pamu_init(void) ret = -ENOMEM; goto error_device_alloc; } - pdev->dev.of_node = of_node_get(np); + + platform_device_set_of_node(pdev, np); ret = pamu_domain_init(); if (ret) @@ -988,17 +990,11 @@ static __init int fsl_pamu_init(void) return 0; error_device_add: - of_node_put(pdev->dev.of_node); - pdev->dev.of_node = NULL; - platform_device_put(pdev); error_device_alloc: platform_driver_unregister(&fsl_of_pamu_driver); -error_driver_register: - of_node_put(np); - return ret; } arch_initcall(fsl_pamu_init); diff --git a/drivers/mfd/tps6586x.c b/drivers/mfd/tps6586x.c index f5f805446603..3cfd2f02b62f 100644 --- a/drivers/mfd/tps6586x.c +++ b/drivers/mfd/tps6586x.c @@ -397,7 +397,7 @@ static int tps6586x_add_subdevs(struct tps6586x *tps6586x, pdev->dev.parent = tps6586x->dev; pdev->dev.platform_data = subdev->platform_data; - pdev->dev.of_node = of_node_get(subdev->of_node); + platform_device_set_of_node(pdev, subdev->of_node); ret = platform_device_add(pdev); if (ret) { diff --git a/drivers/net/ethernet/broadcom/genet/bcmmii.c b/drivers/net/ethernet/broadcom/genet/bcmmii.c index a4e0d5a68268..0f0dbabfaabb 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmmii.c +++ b/drivers/net/ethernet/broadcom/genet/bcmmii.c @@ -490,8 +490,9 @@ static int bcmgenet_mii_register(struct bcmgenet_priv *priv) /* Retain this platform_device pointer for later cleanup */ priv->mii_pdev = ppdev; ppdev->dev.parent = &pdev->dev; + if (dn) - ppdev->dev.of_node = bcmgenet_mii_of_find_mdio(priv); + platform_device_set_of_node(ppdev, bcmgenet_mii_of_find_mdio(priv)); else ppd.phy_mask = ~0; diff --git a/drivers/net/ethernet/marvell/mv643xx_eth.c b/drivers/net/ethernet/marvell/mv643xx_eth.c index 1881583be5ce..9caa1e47c174 100644 --- a/drivers/net/ethernet/marvell/mv643xx_eth.c +++ b/drivers/net/ethernet/marvell/mv643xx_eth.c @@ -2780,7 +2780,7 @@ static int mv643xx_eth_shared_of_add_port(struct platform_device *pdev, goto put_err; } ppdev->dev.coherent_dma_mask = DMA_BIT_MASK(32); - ppdev->dev.of_node = of_node_get(pnp); + platform_device_set_of_node(ppdev, pnp); ret = platform_device_add_resources(ppdev, &res, 1); if (ret) diff --git a/drivers/of/kobj.c b/drivers/of/kobj.c index 1bb61a2c3399..9f17305f112b 100644 --- a/drivers/of/kobj.c +++ b/drivers/of/kobj.c @@ -2,6 +2,8 @@ #include #include +#include + #include "of_private.h" /* true when node is initialized */ @@ -27,6 +29,7 @@ static void of_node_release(struct kobject *kobj) const struct kobj_type of_node_ktype = { .release = of_node_release, }; +EXPORT_SYMBOL_IF_KUNIT(of_node_ktype); static ssize_t of_node_property_read(struct file *filp, struct kobject *kobj, const struct bin_attribute *bin_attr, char *buf, diff --git a/drivers/of/platform.c b/drivers/of/platform.c index 53bca8c6f781..8b1e76407782 100644 --- a/drivers/of/platform.c +++ b/drivers/of/platform.c @@ -126,7 +126,7 @@ struct platform_device *of_device_alloc(struct device_node *np, } /* setup generic device info */ - device_set_node(&dev->dev, of_fwnode_handle(of_node_get(np))); + platform_device_set_of_node(dev, np); dev->dev.parent = parent ? : &platform_bus; if (bus_id) diff --git a/drivers/platform/surface/surface_gpe.c b/drivers/platform/surface/surface_gpe.c index b359413903b1..29b264f2a444 100644 --- a/drivers/platform/surface/surface_gpe.c +++ b/drivers/platform/surface/surface_gpe.c @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -290,9 +291,9 @@ static struct platform_device *surface_gpe_device; static int __init surface_gpe_init(void) { + struct platform_device_info pdevinfo; const struct dmi_system_id *match; struct platform_device *pdev; - struct fwnode_handle *fwnode; int status; match = dmi_first_match(dmi_lid_device_table); @@ -305,44 +306,27 @@ static int __init surface_gpe_init(void) if (status) return status; - fwnode = fwnode_create_software_node(match->driver_data, NULL); - if (IS_ERR(fwnode)) { - status = PTR_ERR(fwnode); - goto err_node; + pdevinfo = (struct platform_device_info){ + .name = "surface_gpe", + .id = PLATFORM_DEVID_NONE, + .properties = match->driver_data, + }; + + pdev = platform_device_register_full(&pdevinfo); + if (IS_ERR(pdev)) { + platform_driver_unregister(&surface_gpe_driver); + return PTR_ERR(pdev); } - pdev = platform_device_alloc("surface_gpe", PLATFORM_DEVID_NONE); - if (!pdev) { - status = -ENOMEM; - goto err_alloc; - } - - pdev->dev.fwnode = fwnode; - - status = platform_device_add(pdev); - if (status) - goto err_add; - surface_gpe_device = pdev; return 0; - -err_add: - platform_device_put(pdev); -err_alloc: - fwnode_remove_software_node(fwnode); -err_node: - platform_driver_unregister(&surface_gpe_driver); - return status; } module_init(surface_gpe_init); static void __exit surface_gpe_exit(void) { - struct fwnode_handle *fwnode = surface_gpe_device->dev.fwnode; - platform_device_unregister(surface_gpe_device); platform_driver_unregister(&surface_gpe_driver); - fwnode_remove_software_node(fwnode); } module_exit(surface_gpe_exit); diff --git a/drivers/pmdomain/imx/gpc.c b/drivers/pmdomain/imx/gpc.c index 42e50c9b4fb9..abca5f449a22 100644 --- a/drivers/pmdomain/imx/gpc.c +++ b/drivers/pmdomain/imx/gpc.c @@ -487,8 +487,7 @@ static int imx_gpc_probe(struct platform_device *pdev) domain->ipg_rate_mhz = ipg_rate_mhz; pd_pdev->dev.parent = &pdev->dev; - pd_pdev->dev.of_node = of_node_get(np); - pd_pdev->dev.fwnode = of_fwnode_handle(np); + platform_device_set_of_node(pd_pdev, np); ret = platform_device_add(pd_pdev); if (ret) { diff --git a/drivers/pwm/pwm_th1520.rs b/drivers/pwm/pwm_th1520.rs index 543bd8dea846..728993998446 100644 --- a/drivers/pwm/pwm_th1520.rs +++ b/drivers/pwm/pwm_th1520.rs @@ -20,7 +20,6 @@ //! this method is not used in this driver. //! -use core::ops::Deref; use kernel::{ clk::Clk, device::{ @@ -28,9 +27,8 @@ Core, Device, // }, - devres, io::{ - mem::IoMem, + mem::DevresIoMem, Io, // }, of, @@ -92,7 +90,7 @@ struct Th1520WfHw { #[pin_data(PinnedDrop)] struct Th1520PwmDriverData { #[pin] - iomem: devres::Devres>, + iomem: DevresIoMem, clk: Clk, } @@ -219,8 +217,7 @@ fn read_waveform( ) -> Result { let data = chip.drvdata(); let hwpwm = pwm.hwpwm(); - let iomem_accessor = data.iomem.access(parent_dev)?; - let iomap = iomem_accessor.deref(); + let iomap = data.iomem.access(parent_dev)?; let ctrl = iomap.try_read32(th1520_pwm_ctrl(hwpwm))?; let period_cycles = iomap.try_read32(th1520_pwm_per(hwpwm))?; @@ -254,8 +251,7 @@ fn write_waveform( ) -> Result { let data = chip.drvdata(); let hwpwm = pwm.hwpwm(); - let iomem_accessor = data.iomem.access(parent_dev)?; - let iomap = iomem_accessor.deref(); + let iomap = data.iomem.access(parent_dev)?; let duty_cycles = iomap.try_read32(th1520_pwm_fp(hwpwm))?; let was_enabled = duty_cycles != 0; @@ -309,7 +305,6 @@ fn drop(self: Pin<&mut Self>) { kernel::of_device_table!( OF_TABLE, - MODULE_OF_TABLE, ::IdInfo, [(of::DeviceId::new(c"thead,th1520-pwm"), ())] ); diff --git a/drivers/reset/reset-rzg2l-usbphy-ctrl.c b/drivers/reset/reset-rzg2l-usbphy-ctrl.c index fd75d9601a3b..f003b360629c 100644 --- a/drivers/reset/reset-rzg2l-usbphy-ctrl.c +++ b/drivers/reset/reset-rzg2l-usbphy-ctrl.c @@ -249,7 +249,7 @@ static int rzg2l_usbphy_ctrl_probe(struct platform_device *pdev) vdev->dev.parent = dev; priv->vdev = vdev; - device_set_of_node_from_dev(&vdev->dev, dev); + platform_device_set_of_node_from_dev(vdev, dev); error = platform_device_add(vdev); if (error) goto err_device_put; diff --git a/drivers/slimbus/qcom-ngd-ctrl.c b/drivers/slimbus/qcom-ngd-ctrl.c index 3071e46d03be..6e89415712ce 100644 --- a/drivers/slimbus/qcom-ngd-ctrl.c +++ b/drivers/slimbus/qcom-ngd-ctrl.c @@ -1539,7 +1539,7 @@ static int of_qcom_slim_ngd_register(struct device *parent, kfree(ngd); return ret; } - ngd->pdev->dev.of_node = of_node_get(node); + platform_device_set_of_node(ngd->pdev, node); ctrl->ngd = ngd; ret = platform_device_add(ngd->pdev); diff --git a/drivers/tty/serdev/Kconfig b/drivers/tty/serdev/Kconfig index 46ae732bfc68..e6dfe949ad01 100644 --- a/drivers/tty/serdev/Kconfig +++ b/drivers/tty/serdev/Kconfig @@ -9,6 +9,13 @@ menuconfig SERIAL_DEV_BUS Note that you typically also want to enable TTY port controller support. +config RUST_SERIAL_DEV_BUS_ABSTRACTIONS + bool "Rust Serial device bus abstractions" + depends on RUST + select SERIAL_DEV_BUS + help + This enables the Rust abstraction for the serial device bus API. + if SERIAL_DEV_BUS config SERIAL_DEV_CTRL_TTYPORT diff --git a/drivers/usb/chipidea/core.c b/drivers/usb/chipidea/core.c index b53672f3ea63..3461dd763a6e 100644 --- a/drivers/usb/chipidea/core.c +++ b/drivers/usb/chipidea/core.c @@ -879,7 +879,7 @@ struct platform_device *ci_hdrc_add_device(struct device *dev, } pdev->dev.parent = dev; - device_set_of_node_from_dev(&pdev->dev, dev); + platform_device_set_of_node_from_dev(pdev, dev); ret = platform_device_add_resources(pdev, res, nres); if (ret) diff --git a/drivers/usb/musb/jz4740.c b/drivers/usb/musb/jz4740.c index df56c972986f..c770ba576f05 100644 --- a/drivers/usb/musb/jz4740.c +++ b/drivers/usb/musb/jz4740.c @@ -273,7 +273,7 @@ static int jz4740_probe(struct platform_device *pdev) musb->dev.parent = dev; musb->dev.dma_mask = &musb->dev.coherent_dma_mask; musb->dev.coherent_dma_mask = DMA_BIT_MASK(32); - device_set_of_node_from_dev(&musb->dev, dev); + platform_device_set_of_node_from_dev(musb, dev); glue->pdev = musb; glue->clk = clk; diff --git a/fs/debugfs/file.c b/fs/debugfs/file.c index edd6aafbfbaa..08de6652a4f3 100644 --- a/fs/debugfs/file.c +++ b/fs/debugfs/file.c @@ -273,7 +273,8 @@ static int debugfs_locked_down(struct inode *inode, (!real_fops || (!real_fops->unlocked_ioctl && !real_fops->compat_ioctl && - !real_fops->mmap))) + !real_fops->mmap && + !real_fops->mmap_prepare))) return 0; if (security_locked_down(LOCKDOWN_DEBUGFS)) diff --git a/fs/debugfs/inode.c b/fs/debugfs/inode.c index 4598142355b9..e054e62919ec 100644 --- a/fs/debugfs/inode.c +++ b/fs/debugfs/inode.c @@ -368,8 +368,11 @@ static struct dentry *debugfs_start_creating(const char *name, if (!debugfs_enabled) return ERR_PTR(-EPERM); - if (!debugfs_initialized()) + if (!debugfs_initialized()) { + pr_err("Unable to create file '%s', debugfs is not initialized yet\n", + name); return ERR_PTR(-ENOENT); + } pr_debug("creating file '%s'\n", name); diff --git a/fs/kernfs/dir.c b/fs/kernfs/dir.c index 97d9d227b66d..82bbaeb326aa 100644 --- a/fs/kernfs/dir.c +++ b/fs/kernfs/dir.c @@ -330,8 +330,8 @@ static unsigned int kernfs_name_hash(const char *name, const struct ns_common *ns) { unsigned long hash = init_name_hash(kernfs_ns_id(ns)); - unsigned int len = strlen(name); - while (len--) + + while (*name) hash = partial_name_hash(*name++, hash); hash = end_name_hash(hash); hash &= 0x7fffffffU; diff --git a/fs/kernfs/inode.c b/fs/kernfs/inode.c index 2cb20294aaf5..237dcdd73fc2 100644 --- a/fs/kernfs/inode.c +++ b/fs/kernfs/inode.c @@ -141,9 +141,9 @@ ssize_t kernfs_iop_listxattr(struct dentry *dentry, char *buf, size_t size) struct kernfs_node *kn = kernfs_dentry_node(dentry); struct kernfs_iattrs *attrs; - attrs = kernfs_iattrs(kn); + attrs = kernfs_iattrs_noalloc(kn); if (!attrs) - return -ENOMEM; + return 0; return simple_xattr_list(d_inode(dentry), &attrs->xattrs, buf, size); } diff --git a/fs/kernfs/symlink.c b/fs/kernfs/symlink.c index 0bd8a2143723..90e2b3221b83 100644 --- a/fs/kernfs/symlink.c +++ b/fs/kernfs/symlink.c @@ -73,7 +73,7 @@ static int kernfs_get_target_path(struct kernfs_node *parent, if ((s - path) + 3 >= PATH_MAX) return -ENAMETOOLONG; - strcpy(s, "../"); + memcpy(s, "../", 4); s += 3; base = kernfs_parent(base); } diff --git a/include/acpi/acpi_bus.h b/include/acpi/acpi_bus.h index 32cac3a6f362..ec3e143caabe 100644 --- a/include/acpi/acpi_bus.h +++ b/include/acpi/acpi_bus.h @@ -162,10 +162,6 @@ struct acpi_hotplug_context { * ----------- */ -bool acpi_of_match_device(const struct acpi_device *adev, - const struct of_device_id *of_match_table, - const struct of_device_id **of_id); - /* Status (_STA) */ struct acpi_device_status { @@ -952,13 +948,6 @@ int acpi_scan_add_dep(acpi_handle handle, struct acpi_handle_list *dep_devices); u32 arch_acpi_add_auto_dep(acpi_handle handle); #else /* CONFIG_ACPI */ -static inline bool acpi_of_match_device(const struct acpi_device *adev, - const struct of_device_id *of_match_table, - const struct of_device_id **of_id) -{ - return false; -} - static inline int register_acpi_bus_type(void *bus) { return 0; } static inline int unregister_acpi_bus_type(void *bus) { return 0; } diff --git a/include/linux/container_of.h b/include/linux/container_of.h index 1f6ebf27d962..28db38e9ee3e 100644 --- a/include/linux/container_of.h +++ b/include/linux/container_of.h @@ -17,11 +17,10 @@ * Do not use container_of() in new code. */ #define container_of(ptr, type, member) ({ \ - void *__mptr = (void *)(ptr); \ - static_assert(__same_type(*(ptr), ((type *)0)->member) || \ + static_assert(__same_type(*(ptr), typeof_member(type, member)) || \ __same_type(*(ptr), void), \ "pointer type mismatch in container_of()"); \ - ((type *)(__mptr - offsetof(type, member))); }) + (type *)((void *)(ptr) - offsetof(type, member)); }) /** * container_of_const - cast a member of a structure out to the containing diff --git a/include/linux/fwnode.h b/include/linux/fwnode.h index 4e86e6990d28..a9dcaf7e7076 100644 --- a/include/linux/fwnode.h +++ b/include/linux/fwnode.h @@ -117,6 +117,8 @@ struct fwnode_reference_args { * @put: Put a reference to an fwnode. * @device_is_available: Return true if the device is available. * @device_get_match_data: Return the device driver match data. + * @device_dma_supported: Return true if DMA is supported. + * @device_get_dma_attr: Return the device DMA attribute. * @property_present: Return true if a property is present. * @property_read_bool: Return a boolean property value. * @property_read_int_array: Read an array of integer properties. Return zero on @@ -134,6 +136,8 @@ struct fwnode_reference_args { * endpoint node. * @graph_get_port_parent: Return the parent node of a port node. * @graph_parse_endpoint: Parse endpoint for port and endpoint id. + * @iomap: Map the I/O memory of a given index for a fwnode. + * @irq_get: Get the IRQ of a given index for a fwnode. * @add_links: Create fwnode links to all the suppliers of the fwnode. Return * zero on success, a negative error code otherwise. */ diff --git a/include/linux/kobject.h b/include/linux/kobject.h index bcb5d4e32001..55e37a5d405e 100644 --- a/include/linux/kobject.h +++ b/include/linux/kobject.h @@ -138,12 +138,79 @@ struct kset_uevent_ops { struct kobj_attribute { struct attribute attr; - ssize_t (*show)(struct kobject *kobj, struct kobj_attribute *attr, - char *buf); - ssize_t (*store)(struct kobject *kobj, struct kobj_attribute *attr, - const char *buf, size_t count); + __SYSFS_FUNCTION_ALTERNATIVE( + ssize_t (*show)(struct kobject *kobj, struct kobj_attribute *attr, char *buf); + ssize_t (*show_const)(struct kobject *kobj, const struct kobj_attribute *attr, + char *buf); + ); + __SYSFS_FUNCTION_ALTERNATIVE( + ssize_t (*store)(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count); + ssize_t (*store_const)(struct kobject *kobj, const struct kobj_attribute *attr, + const char *buf, size_t count); + ); }; +typedef ssize_t __kobj_show_handler_const(struct kobject *kobj, const struct kobj_attribute *attr, + char *buf); +typedef ssize_t __kobj_store_handler_const(struct kobject *kobj, const struct kobj_attribute *attr, + const char *buf, size_t count); + +#ifdef CONFIG_CFI + +#define __KOBJ_ATTR_SHOW_STORE(_show, _store) \ + .show = _Generic(_show, \ + __kobj_show_handler_const * : NULL, \ + default : _show \ + ), \ + .show_const = _Generic(_show, \ + __kobj_show_handler_const * : _show, \ + default : NULL \ + ), \ + .store = _Generic(_store, \ + __kobj_store_handler_const * : NULL, \ + default : _store \ + ), \ + .store_const = _Generic(_store, \ + __kobj_store_handler_const * : _store, \ + default : NULL \ + ), + +#else + +#define __KOBJ_ATTR_SHOW_STORE(_show, _store) \ + .show = _Generic(_show, \ + __kobj_show_handler_const * : (void *)_show, \ + default : _show \ + ), \ + .store = _Generic(_store, \ + __kobj_store_handler_const * : (void *)_store, \ + default : _store \ + ), \ + +#endif + +#define __KOBJ_ATTR(_name, _mode, _show, _store) { \ + .attr = { .name = __stringify(_name), \ + .mode = VERIFY_OCTAL_PERMISSIONS(_mode) }, \ + __KOBJ_ATTR_SHOW_STORE(_show, _store) \ +} + +#define __KOBJ_ATTR_RO_MODE(_name, _mode) \ + __KOBJ_ATTR(_name, _mode, _name##_show, NULL) + +#define __KOBJ_ATTR_RO(_name) \ + __KOBJ_ATTR_RO_MODE(_name, 0444) + +#define __KOBJ_ATTR_RW_MODE(_name, _mode) \ + __KOBJ_ATTR(_name, _mode, _name##_show, _name##_store) + +#define __KOBJ_ATTR_WO(_name) \ + __KOBJ_ATTR(_name, 0200, NULL, _name##_store) + +#define __KOBJ_ATTR_RW(_name) \ + __KOBJ_ATTR(_name, 0644, _name##_show, _name##_store) + extern const struct sysfs_ops kobj_sysfs_ops; struct sock; diff --git a/include/linux/pci.h b/include/linux/pci.h index 64b308b6e61c..3d2c1ac645ff 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -1783,6 +1783,26 @@ void pci_free_irq_vectors(struct pci_dev *dev); int pci_irq_vector(struct pci_dev *dev, unsigned int nr); const struct cpumask *pci_irq_get_affinity(struct pci_dev *pdev, int vec); +/** + * pci_irq_type - Get the interrupt type of a PCI device + * @pdev: the PCI device to operate on + * + * Discriminate the interrupt type the PCI core selected for this device + * after a successful pci_alloc_irq_vectors() call. + * + * Return: %PCI_IRQ_MSIX, %PCI_IRQ_MSI, or %PCI_IRQ_INTX. + */ +static inline unsigned int pci_irq_type(struct pci_dev *pdev) +{ + if (pdev->msix_enabled) + return PCI_IRQ_MSIX; + + if (pdev->msi_enabled) + return PCI_IRQ_MSI; + + return PCI_IRQ_INTX; +} + #else static inline int pci_msi_vec_count(struct pci_dev *dev) { return -ENOSYS; } static inline void pci_disable_msi(struct pci_dev *dev) { } @@ -1845,6 +1865,11 @@ static inline const struct cpumask *pci_irq_get_affinity(struct pci_dev *pdev, { return cpu_possible_mask; } + +static inline unsigned int pci_irq_type(struct pci_dev *pdev) +{ + return PCI_IRQ_INTX; +} #endif /** @@ -2255,6 +2280,11 @@ static inline bool pci_suspend_retains_context(struct pci_dev *pdev) { return true; } + +static inline unsigned int pci_irq_type(struct pci_dev *pdev) +{ + return 0; +} #endif /* CONFIG_PCI */ /* Include architecture-dependent settings and functions */ diff --git a/include/linux/platform_device.h b/include/linux/platform_device.h index 8c566f09d04e..3d5bbcbae730 100644 --- a/include/linux/platform_device.h +++ b/include/linux/platform_device.h @@ -19,6 +19,8 @@ struct irq_affinity; struct mfd_cell; struct property_entry; +struct device_node; +struct fwnode_handle; struct platform_device { const char *name; @@ -262,6 +264,12 @@ extern int platform_device_add_resources(struct platform_device *pdev, unsigned int num); extern int platform_device_add_data(struct platform_device *pdev, const void *data, size_t size); +void platform_device_set_of_node(struct platform_device *pdev, + struct device_node *np); +void platform_device_set_fwnode(struct platform_device *pdev, + struct fwnode_handle *fwnode); +void platform_device_set_of_node_from_dev(struct platform_device *pdev, + const struct device *dev2); extern int platform_device_add(struct platform_device *pdev); extern void platform_device_del(struct platform_device *pdev); extern void platform_device_put(struct platform_device *pdev); diff --git a/include/linux/property.h b/include/linux/property.h index 14c304db4664..907c790a3f01 100644 --- a/include/linux/property.h +++ b/include/linux/property.h @@ -397,11 +397,13 @@ struct property_entry { union { const void *pointer; union { + /* private: internal representation of @value */ u8 u8_data[sizeof(u64) / sizeof(u8)]; u16 u16_data[sizeof(u64) / sizeof(u16)]; u32 u32_data[sizeof(u64) / sizeof(u32)]; u64 u64_data[sizeof(u64) / sizeof(u64)]; const char *str[sizeof(u64) / sizeof(char *)]; + /* public: */ } value; }; }; diff --git a/include/linux/sys_soc.h b/include/linux/sys_soc.h index f19f5cec18e2..cedabf177f47 100644 --- a/include/linux/sys_soc.h +++ b/include/linux/sys_soc.h @@ -21,19 +21,26 @@ struct soc_device_attribute { /** * soc_device_register - register SoC as a device * @soc_plat_dev_attr: Attributes passed from platform to be attributed to a SoC + * + * Returns: + * - %NULL if the SoC bus is not yet registered; + * - on success, the newly allocated &struct soc_device pointer; + * - on failure, a negative error code as an ERR_PTR(). */ struct soc_device *soc_device_register( struct soc_device_attribute *soc_plat_dev_attr); /** * soc_device_unregister - unregister SoC device - * @dev: SoC device to be unregistered + * @soc_dev: SoC device to be unregistered */ void soc_device_unregister(struct soc_device *soc_dev); /** * soc_device_to_device - helper function to fetch struct device * @soc: Previously registered SoC device container + * + * Returns: &struct device pointer for this @soc */ struct device *soc_device_to_device(struct soc_device *soc); diff --git a/lib/kobject.c b/lib/kobject.c index 9c9ff0f5175f..e7b010a989fb 100644 --- a/lib/kobject.c +++ b/lib/kobject.c @@ -823,9 +823,11 @@ static ssize_t kobj_attr_show(struct kobject *kobj, struct attribute *attr, struct kobj_attribute *kattr; ssize_t ret = -EIO; - kattr = container_of(attr, struct kobj_attribute, attr); + kattr = container_of_const(attr, struct kobj_attribute, attr); if (kattr->show) ret = kattr->show(kobj, kattr, buf); + else if (kattr->show_const) + ret = kattr->show_const(kobj, kattr, buf); return ret; } @@ -835,9 +837,11 @@ static ssize_t kobj_attr_store(struct kobject *kobj, struct attribute *attr, struct kobj_attribute *kattr; ssize_t ret = -EIO; - kattr = container_of(attr, struct kobj_attribute, attr); + kattr = container_of_const(attr, struct kobj_attribute, attr); if (kattr->store) ret = kattr->store(kobj, kattr, buf, count); + else if (kattr->store_const) + ret = kattr->store_const(kobj, kattr, buf, count); return ret; } diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 1124785e210b..fe7c505da236 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -85,6 +85,7 @@ #include #include #include +#include #include #include #include diff --git a/rust/helpers/acpi.c b/rust/helpers/acpi.c deleted file mode 100644 index e75c9807bbad..000000000000 --- a/rust/helpers/acpi.c +++ /dev/null @@ -1,16 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include -#include - -__rust_helper bool rust_helper_acpi_of_match_device(const struct acpi_device *adev, - const struct of_device_id *of_match_table, - const struct of_device_id **of_id) -{ - return acpi_of_match_device(adev, of_match_table, of_id); -} - -__rust_helper struct acpi_device *rust_helper_to_acpi_device_node(struct fwnode_handle *fwnode) -{ - return to_acpi_device_node(fwnode); -} diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index d60af162a078..198c5f807eba 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -38,7 +38,6 @@ #define __rust_helper __always_inline #endif -#include "acpi.c" #include "atomic.c" #include "atomic_ext.c" #include "auxiliary.c" @@ -90,6 +89,7 @@ #include "regulator.c" #include "scatterlist.c" #include "security.c" +#include "serdev.c" #include "signal.c" #include "slab.c" #include "spinlock.c" diff --git a/rust/helpers/io.c b/rust/helpers/io.c index 1edbc274951c..308950aae19c 100644 --- a/rust/helpers/io.c +++ b/rust/helpers/io.c @@ -21,6 +21,19 @@ __rust_helper void rust_helper_iounmap(void __iomem *addr) } #endif /* CONFIG_HAS_IOMEM */ +__rust_helper void rust_helper_memcpy_fromio(void *dst, + const volatile void __iomem *src, + size_t count) +{ + memcpy_fromio(dst, src, count); +} + +__rust_helper void rust_helper_memcpy_toio(volatile void __iomem *dst, + const void *src, size_t count) +{ + memcpy_toio(dst, src, count); +} + __rust_helper u8 rust_helper_readb(const void __iomem *addr) { return readb(addr); diff --git a/rust/helpers/pci.c b/rust/helpers/pci.c index e44905317d75..23b06becb448 100644 --- a/rust/helpers/pci.c +++ b/rust/helpers/pci.c @@ -24,6 +24,11 @@ __rust_helper bool rust_helper_dev_is_pci(const struct device *dev) return dev_is_pci(dev); } +__rust_helper unsigned int rust_helper_pci_irq_type(struct pci_dev *pdev) +{ + return pci_irq_type(pdev); +} + #ifndef CONFIG_PCI_MSI __rust_helper int rust_helper_pci_alloc_irq_vectors(struct pci_dev *dev, unsigned int min_vecs, diff --git a/rust/helpers/serdev.c b/rust/helpers/serdev.c new file mode 100644 index 000000000000..c52b78ca3fc7 --- /dev/null +++ b/rust/helpers/serdev.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper +void rust_helper_serdev_device_driver_unregister(struct serdev_device_driver *sdrv) +{ + serdev_device_driver_unregister(sdrv); +} + +__rust_helper +void rust_helper_serdev_device_put(struct serdev_device *serdev) +{ + serdev_device_put(serdev); +} + +__rust_helper +void rust_helper_serdev_device_set_client_ops(struct serdev_device *serdev, + const struct serdev_device_ops *ops) +{ + serdev_device_set_client_ops(serdev, ops); +} diff --git a/rust/kernel/acpi.rs b/rust/kernel/acpi.rs index 9b8efa623130..ea2ce61ee393 100644 --- a/rust/kernel/acpi.rs +++ b/rust/kernel/acpi.rs @@ -25,10 +25,6 @@ unsafe impl RawDeviceId for DeviceId { // SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. unsafe impl RawDeviceIdIndex for DeviceId { const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::acpi_device_id, driver_data); - - fn index(&self) -> usize { - self.0.driver_data - } } impl DeviceId { @@ -53,13 +49,7 @@ pub const fn new(id: &'static CStr) -> Self { /// Create an ACPI `IdTable` with an "alias" for modpost. #[macro_export] macro_rules! acpi_device_table { - ($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => { - const $table_name: $crate::device_id::IdArray< - $crate::acpi::DeviceId, - $id_info_type, - { $table_data.len() }, - > = $crate::device_id::IdArray::new($table_data); - - $crate::module_device_table!("acpi", $module_table_name, $table_name); + ($($tt:tt)*) => { + $crate::module_device_table!("acpi", $crate::acpi::DeviceId, $($tt)*); }; } diff --git a/rust/kernel/auxiliary.rs b/rust/kernel/auxiliary.rs index cc9745fbf179..60dfbec8f330 100644 --- a/rust/kernel/auxiliary.rs +++ b/rust/kernel/auxiliary.rs @@ -20,6 +20,7 @@ }, prelude::*, types::{ + CovariantForLt, ForLt, ForeignOwnable, Opaque, // @@ -93,7 +94,9 @@ extern "C" fn probe_callback( // SAFETY: `DeviceId` is a `#[repr(transparent)`] wrapper of `struct auxiliary_device_id` // and does not add additional invariants, so it's safe to transmute. let id = unsafe { &*id.cast::() }; - let info = T::ID_TABLE.info(id.index()); + + // SAFETY: `id` comes from `T::ID_TABLE` which is of type `IdArray<_, T::IdInfo>`. + let info = unsafe { id.info_unchecked::() }; from_result(|| { let data = T::probe(adev, info); @@ -169,10 +172,6 @@ unsafe impl RawDeviceId for DeviceId { unsafe impl RawDeviceIdIndex for DeviceId { const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::auxiliary_device_id, driver_data); - - fn index(&self) -> usize { - self.0.driver_data - } } /// IdTable type for auxiliary drivers. @@ -181,14 +180,8 @@ fn index(&self) -> usize { /// Create a auxiliary `IdTable` with its alias for modpost. #[macro_export] macro_rules! auxiliary_device_table { - ($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => { - const $table_name: $crate::device_id::IdArray< - $crate::auxiliary::DeviceId, - $id_info_type, - { $table_data.len() }, - > = $crate::device_id::IdArray::new($table_data); - - $crate::module_device_table!("auxiliary", $module_table_name, $table_name); + ($($tt:tt)*) => { + $crate::module_device_table!("auxiliary", $crate::auxiliary::DeviceId, $($tt)*); }; } @@ -270,18 +263,15 @@ pub fn parent(&self) -> &device::Device { unsafe { parent.as_bound() } } - /// Returns a pinned reference to the registration data set by the registering (parent) driver. + /// Returns the stored registration data as a pinned reference. /// - /// `F` is the [`ForLt`](trait@ForLt) encoding of the data type. The returned - /// reference has its lifetime shortened from `'static` to `&self`'s borrow lifetime via - /// [`ForLt::cast_ref`]. + /// Performs null and [`TypeId`] checks, then borrows the stored [`KBox`]. /// - /// Returns [`EINVAL`] if `F` does not match the type used by the parent driver when calling - /// [`Registration::new()`]. + /// # Safety /// - /// Returns [`ENOENT`] if no registration data has been set, e.g. when the device was - /// registered by a C driver. - pub fn registration_data(&self) -> Result>> { + /// Callers must ensure that the lifetime shortening from the original `'static` storage to + /// `'_` is sound, e.g. via an HRTB closure or [`CovariantForLt`] guarantee. + unsafe fn registration_data_pinned(&self) -> Result>> { // SAFETY: By the type invariant, `self.as_raw()` is a valid `struct auxiliary_device`. let ptr = unsafe { (*self.as_raw()).registration_data_rust }; if ptr.is_null() { @@ -300,17 +290,59 @@ pub fn registration_data(&self) -> Result>> { return Err(EINVAL); } - // SAFETY: The `TypeId` check above confirms that the stored type matches - // `F::Of<'static>`; `ptr` remains valid until `Registration::drop()` calls - // `from_foreign()`. - let wrapper = unsafe { Pin::>>>::borrow(ptr) }; + // SAFETY: The `TypeId` check above confirms that the stored type matches `F`'s + // encoding; lifetimes are erased at runtime, so borrowing as `F::Of<'_>` is + // layout-compatible with the stored `F::Of<'static>`. `ptr` remains valid until + // `Registration::drop()` calls `from_foreign()`. + let wrapper = unsafe { Pin::>>>::borrow(ptr) }; // SAFETY: `data` is a structurally pinned field of `RegistrationData`. - let pinned: Pin<&F::Of<'_>> = unsafe { wrapper.map_unchecked(|w| &w.data) }; + Ok(unsafe { wrapper.map_unchecked(|w| &w.data) }) + } - // SAFETY: The data was pinned when stored; `cast_ref` only shortens - // the lifetime, so the pinning guarantee is preserved. - Ok(unsafe { Pin::new_unchecked(F::cast_ref(pinned.get_ref())) }) + /// Access the registration data set by the registering (parent) driver through a closure. + /// + /// `F` is the [`ForLt`](trait@ForLt) encoding of the data type. The closure receives a pinned + /// reference to the registration data. + /// + /// For covariant types that implement [`trait@CovariantForLt`], prefer + /// [`registration_data`](Self::registration_data) which returns a direct reference. + /// + /// Returns [`EINVAL`] if `F` does not match the type used by the parent driver when calling + /// [`Registration::new()`]. + /// + /// Returns [`ENOENT`] if no registration data has been set, e.g. when the device was + /// registered by a C driver. + #[inline] + pub fn registration_data_with( + &self, + f: impl for<'a> FnOnce(Pin<&'a F::Of<'a>>) -> R, + ) -> Result { + // SAFETY: The HRTB closure prevents the caller from smuggling in references with a + // concrete short lifetime, making the round-trip from `'static` sound regardless of + // variance. + let pinned = unsafe { self.registration_data_pinned::()? }; + + Ok(f(pinned)) + } + + /// Returns a pinned reference to the registration data set by the registering (parent) driver. + /// + /// This method is only available when `F` implements [`trait@CovariantForLt`], which guarantees + /// that the lifetime shortening is sound. + /// + /// For non-covariant types, use the closure-based [`Self::registration_data_with`]. + /// + /// Returns [`EINVAL`] if `F` does not match the type used by the parent driver when calling + /// [`Registration::new()`]. + /// + /// Returns [`ENOENT`] if no registration data has been set, e.g. when the device was + /// registered by a C driver. + #[inline] + pub fn registration_data(&self) -> Result>> { + // SAFETY: `CovariantForLt` guarantees covariance, which makes the lifetime shortening + // from `'static` to `'_` performed by `registration_data_pinned` sound. + unsafe { self.registration_data_pinned::() } } } @@ -401,7 +433,9 @@ struct RegistrationData { /// /// The type parameter `F` is a [`ForLt`](trait@ForLt) encoding of the registration /// data type. For non-lifetime-parameterized types, use [`ForLt!(T)`](macro@ForLt). -/// The data can be accessed by the auxiliary driver through [`Device::registration_data()`]. +/// +/// The data can be accessed by the auxiliary driver through [`Device::registration_data()`] and +/// [`Device::registration_data_with()`]. /// /// # Invariants /// diff --git a/rust/kernel/debugfs/traits.rs b/rust/kernel/debugfs/traits.rs index 8c39524b6a99..b295f8420abd 100644 --- a/rust/kernel/debugfs/traits.rs +++ b/rust/kernel/debugfs/traits.rs @@ -18,10 +18,6 @@ Arc, Mutex, // }, - transmute::{ - AsBytes, - FromBytes, // - }, uaccess::{ UserSliceReader, UserSliceWriter, // @@ -36,6 +32,8 @@ str::FromStr, }; +use zerocopy::Immutable; + /// A trait for types that can be written into a string. /// /// This works very similarly to `Debug`, and is automatically implemented if `Debug` is @@ -76,8 +74,8 @@ fn write_to_slice( ) -> Result; } -// Base implementation for any `T: AsBytes`. -impl BinaryWriter for T { +// Base implementation for any `T: Immutable + IntoBytes`. +impl BinaryWriter for T { fn write_to_slice( &self, writer: &mut UserSliceWriter, @@ -147,7 +145,7 @@ fn write_to_slice( // Delegate for `Vec`. impl BinaryWriter for Vec where - T: AsBytes, + T: Immutable + IntoBytes, A: Allocator, { fn write_to_slice( @@ -155,14 +153,7 @@ fn write_to_slice( writer: &mut UserSliceWriter, offset: &mut file::Offset, ) -> Result { - let slice = self.as_slice(); - - // SAFETY: `T: AsBytes` allows us to treat `&[T]` as `&[u8]`. - let buffer = unsafe { - core::slice::from_raw_parts(slice.as_ptr().cast(), core::mem::size_of_val(slice)) - }; - - writer.write_slice_file(buffer, offset) + writer.write_slice_file(self.as_bytes(), offset) } } @@ -230,14 +221,14 @@ fn read_from_slice_mut( ) -> Result; } -// Base implementation for any `T: AsBytes + FromBytes`. -impl BinaryReaderMut for T { +// Base implementation for any `T: FromBytes + IntoBytes`. +impl BinaryReaderMut for T { fn read_from_slice_mut( &mut self, reader: &mut UserSliceReader, offset: &mut file::Offset, ) -> Result { - reader.read_slice_file(self.as_bytes_mut(), offset) + reader.read_slice_file(self.as_mut_bytes(), offset) } } @@ -255,7 +246,7 @@ fn read_from_slice_mut( // Delegate for `Vec`: Support a `Vec` with an outer lock. impl BinaryReaderMut for Vec where - T: AsBytes + FromBytes, + T: FromBytes + IntoBytes, A: Allocator, { fn read_from_slice_mut( @@ -263,17 +254,7 @@ fn read_from_slice_mut( reader: &mut UserSliceReader, offset: &mut file::Offset, ) -> Result { - let slice = self.as_mut_slice(); - - // SAFETY: `T: AsBytes + FromBytes` allows us to treat `&mut [T]` as `&mut [u8]`. - let buffer = unsafe { - core::slice::from_raw_parts_mut( - slice.as_mut_ptr().cast(), - core::mem::size_of_val(slice), - ) - }; - - reader.read_slice_file(buffer, offset) + reader.read_slice_file(self.as_mut_bytes(), offset) } } diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index 1a38b3bbdfb7..2291d85b6849 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -54,7 +54,8 @@ /// reference is valid in. For instance, the [`Bound`] context guarantees that the [`Device`] is /// bound to a driver for the entire duration of the existence of a [`Device`] reference. /// -/// Other [`DeviceContext`] types besides [`Bound`] are [`Normal`], [`Core`] and [`CoreInternal`]. +/// Other [`DeviceContext`] types besides [`Bound`] are [`Normal`], [`Core`], [`CoreInternal`] and +/// [`BoundInternal`]. /// /// Unless selected otherwise [`Device`] defaults to the [`Normal`] [`DeviceContext`], which by /// itself has no additional requirements. @@ -235,7 +236,9 @@ pub(crate) unsafe fn drvdata_obtain(&self) -> Option>> { // in `into_foreign()`. Some(unsafe { Pin::>::from_foreign(ptr.cast()) }) } +} +impl Device { /// Borrow the driver's private data bound to this [`Device`]. /// /// # Safety @@ -245,22 +248,6 @@ pub(crate) unsafe fn drvdata_obtain(&self) -> Option>> { /// - The type `T` must match the type of the `ForeignOwnable` previously stored by /// [`Device::set_drvdata`]. pub unsafe fn drvdata_borrow(&self) -> Pin<&T> { - // SAFETY: `drvdata_unchecked()` has the exact same safety requirements as the ones - // required by this method. - unsafe { self.drvdata_unchecked() } - } -} - -impl Device { - /// Borrow the driver's private data bound to this [`Device`]. - /// - /// # Safety - /// - /// - Must only be called after a preceding call to [`Device::set_drvdata`] and before - /// the device is fully unbound. - /// - The type `T` must match the type of the `ForeignOwnable` previously stored by - /// [`Device::set_drvdata`]. - unsafe fn drvdata_unchecked(&self) -> Pin<&T> { // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`. let ptr = unsafe { bindings::dev_get_drvdata(self.as_raw()) }; @@ -476,7 +463,8 @@ unsafe impl Sync for Device {} /// [`DeviceContext`] is a marker trait for types representing the context of a bus specific /// [`Device`]. /// -/// The specific device context types are: [`CoreInternal`], [`Core`], [`Bound`] and [`Normal`]. +/// The specific device context types are: [`CoreInternal`], [`Core`], [`BoundInternal`], [`Bound`] +/// and [`Normal`]. /// /// [`DeviceContext`] types are hierarchical, which means that there is a strict hierarchy that /// defines which [`DeviceContext`] type can be derived from another. For instance, any @@ -485,6 +473,11 @@ unsafe impl Sync for Device {} /// The following enumeration illustrates the dereference hierarchy of [`DeviceContext`] types. /// /// - [`CoreInternal`] => [`Core`] => [`Bound`] => [`Normal`] +/// - [`BoundInternal`] => [`Bound`] => [`Normal`] +/// +/// Both [`CoreInternal`] and [`BoundInternal`] implement the [`InternalBoundContext`] trait, +/// which provides access to internal bus abstraction methods on [`Device`] that are not available +/// to drivers. /// /// Bus devices can automatically implement the dereference hierarchy by using /// [`impl_device_context_deref`]. @@ -511,7 +504,11 @@ pub trait DeviceContext: private::Sealed {} /// callback it appears in. It is intended to be used for synchronization purposes. Bus device /// implementations can implement methods for [`Device`], such that they can only be called /// from bus callbacks. -pub struct Core<'a>(PhantomData<&'a ()>); +/// +/// The lifetime `'a` is for "lifetime branding" purpose. Callbacks need to polymorphic over this +/// lifetime so the `&'bound Device>` provided to them cannot outlive the scope of the +/// function. For this reason, it needs to be invariant. +pub struct Core<'a>(PhantomData &'a ()>); /// Semantically the same as [`Core`], but reserved for internal usage of the corresponding bus /// abstraction. @@ -522,7 +519,21 @@ pub trait DeviceContext: private::Sealed {} /// /// This context mainly exists to share generic [`Device`] infrastructure that should only be called /// from bus callbacks with bus abstractions, but without making them accessible for drivers. -pub struct CoreInternal<'a>(PhantomData<&'a ()>); +/// +/// Lifetime `'a` is invariant for the same reason as [`Core`]. +pub struct CoreInternal<'a>(PhantomData &'a ()>); + +/// Semantically the same as [`Bound`], but reserved for internal usage of the corresponding bus +/// abstraction. +/// +/// The internal bound context is intended to be used in exactly the same way as the [`Bound`] +/// context, with the difference that this [`DeviceContext`] is internal to the corresponding bus +/// abstraction. +/// +/// This context exists for cases where the bus abstraction needs access to internal device +/// infrastructure (such as [`Device::drvdata_borrow`]), where [`CoreInternal`] would not be +/// justified. +pub struct BoundInternal; /// The [`Bound`] context is the [`DeviceContext`] of a bus specific device when it is guaranteed to /// be bound to a driver. @@ -546,16 +557,28 @@ mod private { pub trait Sealed {} impl Sealed for super::Bound {} + impl Sealed for super::BoundInternal {} impl<'a> Sealed for super::Core<'a> {} impl<'a> Sealed for super::CoreInternal<'a> {} impl Sealed for super::Normal {} } impl DeviceContext for Bound {} +impl DeviceContext for BoundInternal {} impl<'a> DeviceContext for Core<'a> {} impl<'a> DeviceContext for CoreInternal<'a> {} impl DeviceContext for Normal {} +/// Marker trait for [`DeviceContext`] types that have internal bound-level access. +/// +/// This trait is implemented by [`CoreInternal`] and [`BoundInternal`], allowing methods that +/// require internal bus abstraction access to a bound device to be generic over both contexts. +/// +/// Methods bounded by this trait are available to bus abstractions but not to drivers. +pub trait InternalBoundContext: DeviceContext {} +impl<'a> InternalBoundContext for CoreInternal<'a> {} +impl InternalBoundContext for BoundInternal {} + impl AsRef> for Device { #[inline] fn as_ref(&self) -> &Device { @@ -661,6 +684,13 @@ macro_rules! impl_device_context_deref { <'a> $crate::device::Core<'a> => $crate::device::Bound }); + // SAFETY: This macro has the exact same safety requirement as + // `__impl_device_context_deref!`. + ::kernel::__impl_device_context_deref!(unsafe { + $device, + $crate::device::BoundInternal => $crate::device::Bound + }); + // SAFETY: This macro has the exact same safety requirement as // `__impl_device_context_deref!`. ::kernel::__impl_device_context_deref!(unsafe { @@ -700,6 +730,7 @@ macro_rules! impl_device_context_into_aref { ::kernel::__impl_device_context_into_aref!( <'a> $crate::device::Core<'a>, $device ); + ::kernel::__impl_device_context_into_aref!($crate::device::BoundInternal, $device); ::kernel::__impl_device_context_into_aref!($crate::device::Bound, $device); }; } diff --git a/rust/kernel/device_id.rs b/rust/kernel/device_id.rs index 8e9721446014..c81fca5b4986 100644 --- a/rust/kernel/device_id.rs +++ b/rust/kernel/device_id.rs @@ -5,7 +5,10 @@ //! Each bus / subsystem that matches device and driver through a bus / subsystem specific ID is //! expected to implement [`RawDeviceId`]. -use core::mem::MaybeUninit; +use core::{ + marker::PhantomData, + mem::MaybeUninit, // +}; /// Marker trait to indicate a Rust device ID type represents a corresponding C device ID type. /// @@ -47,112 +50,110 @@ pub unsafe trait RawDeviceIdIndex: RawDeviceId { /// The offset (in bytes) to the context/data field in the raw device ID. const DRIVER_DATA_OFFSET: usize; - /// The index stored at `DRIVER_DATA_OFFSET` of the implementor of the [`RawDeviceIdIndex`] - /// trait. - fn index(&self) -> usize; -} + /// Obtain the data pointer stored inside the device ID. + /// + /// # Safety + /// + /// `&Self` must be stored inside a `IdArray`. + unsafe fn info_unchecked(&self) -> &'static U { + // SAFETY: By safety requirement of the trait, this is `self.driver_data as *const U` and by + // the safety requirement of the function, this is stored in `IdArray` so is + // convertible to `&'static U`. + unsafe { + core::ptr::from_ref(self) + .byte_add(Self::DRIVER_DATA_OFFSET) + .cast::<&U>() + .read() + } + } -/// A zero-terminated device id array. -#[repr(C)] -pub struct RawIdArray { - ids: [T::RawType; N], - sentinel: MaybeUninit, -} - -impl RawIdArray { - #[doc(hidden)] - pub const fn size(&self) -> usize { - core::mem::size_of::() + /// Obtain the data pointer stored inside the device ID. + /// + /// # Safety + /// + /// `&Self` must be stored inside a `IdArray`, or has NULL (or 0) as driver data. + unsafe fn info_unchecked_opt(&self) -> Option<&'static U> { + // SAFETY: By safety requirement of the trait, this is `self.driver_data as *const U` and by + // the safety requirement of the function, if this is stored in `IdArray`, this is + // convertible to `Option<&'static U>`. Otherwise it is NULL which is `None` as + // `Option<&U>`. + unsafe { + core::ptr::from_ref(self) + .byte_add(Self::DRIVER_DATA_OFFSET) + .cast::>() + .read() + } } } /// A zero-terminated device id array, followed by context data. #[repr(C)] -pub struct IdArray { - raw_ids: RawIdArray, - id_infos: [U; N], +pub struct IdArray { + // This is `MaybeUninit` so any bytes inside it can carry provenance in CTFE. + // If this were `T::RawType`, integer fields would not be able to contain pointers. + ids: [MaybeUninit; N], + sentinel: MaybeUninit, + phantom: PhantomData<&'static U>, } -impl IdArray { +// SAFETY: device ID is plain data plus a `&'static U` and can thus be sent between threads safely +// if `&U` can. +unsafe impl Send for IdArray {} + +// SAFETY: device ID is plain data plus a `&'static U` and can thus be shared between threads safely +// if `&U` can. +unsafe impl Sync for IdArray {} + +impl IdArray { /// Creates a new instance of the array. /// /// The contents are derived from the given identifiers and context information. - /// - /// # Safety - /// - /// `data_offset` as `None` is always safe. - /// If `data_offset` is `Some(data_offset)`, then: - /// - `data_offset` must be the correct offset (in bytes) to the context/data field - /// (e.g., the `driver_data` field) within the raw device ID structure. - /// - The field at `data_offset` must be correctly sized to hold a `usize`. - const unsafe fn build(ids: [(T, U); N], data_offset: Option) -> Self { + pub const fn new(ids: [(T, &'static U); N]) -> Self { let mut raw_ids = [const { MaybeUninit::::uninit() }; N]; - let mut infos = [const { MaybeUninit::uninit() }; N]; let mut i = 0usize; while i < N { // SAFETY: by the safety requirement of `RawDeviceId`, we're guaranteed that `T` is // layout-wise compatible with `RawType`. raw_ids[i] = unsafe { core::mem::transmute_copy(&ids[i].0) }; - if let Some(data_offset) = data_offset { - // SAFETY: by the safety requirement of this function, this would be effectively - // `raw_ids[i].driver_data = i;`. - unsafe { - raw_ids[i] - .as_mut_ptr() - .byte_add(data_offset) - .cast::() - .write(i); - } + // SAFETY: by the safety requirement of `RawDeviceIdIndex`, this would be effectively + // `raw_ids[i].driver_data = ids[i].1;`. + unsafe { + raw_ids[i] + .as_mut_ptr() + .byte_add(T::DRIVER_DATA_OFFSET) + .cast::<&U>() + .write(ids[i].1); } - // SAFETY: this is effectively a move: `infos[i] = ids[i].1`. We make a copy here but - // later forget `ids`. - infos[i] = MaybeUninit::new(unsafe { core::ptr::read(&ids[i].1) }); i += 1; } core::mem::forget(ids); Self { - raw_ids: RawIdArray { - // SAFETY: this is effectively `array_assume_init`, which is unstable, so we use - // `transmute_copy` instead. We have initialized all elements of `raw_ids` so this - // `array_assume_init` is safe. - ids: unsafe { core::mem::transmute_copy(&raw_ids) }, - sentinel: MaybeUninit::zeroed(), - }, - // SAFETY: We have initialized all elements of `infos` so this `array_assume_init` is - // safe. - id_infos: unsafe { core::mem::transmute_copy(&infos) }, + ids: raw_ids, + sentinel: MaybeUninit::zeroed(), + phantom: PhantomData, } } +} +impl IdArray { /// Creates a new instance of the array without writing index values. /// /// The contents are derived from the given identifiers and context information. /// If the device implements [`RawDeviceIdIndex`], consider using [`IdArray::new`] instead. - pub const fn new_without_index(ids: [(T, U); N]) -> Self { - // SAFETY: Calling `Self::build` with `offset = None` is always safe, - // because no raw memory writes are performed in this case. - unsafe { Self::build(ids, None) } - } + pub const fn new_without_index(ids: [T; N]) -> Self { + // SAFETY: `T` is layout-wise compatible with `T::RawType`, so is the array of them. + let raw_ids: [MaybeUninit; N] = unsafe { core::mem::transmute_copy(&ids) }; + core::mem::forget(ids); - /// Reference to the contained [`RawIdArray`]. - pub const fn raw_ids(&self) -> &RawIdArray { - &self.raw_ids - } -} - -impl IdArray { - /// Creates a new instance of the array. - /// - /// The contents are derived from the given identifiers and context information. - pub const fn new(ids: [(T, U); N]) -> Self { - // SAFETY: by the safety requirement of `RawDeviceIdIndex`, - // `T::DRIVER_DATA_OFFSET` is guaranteed to be the correct offset (in bytes) to - // a field within `T::RawType`. - unsafe { Self::build(ids, Some(T::DRIVER_DATA_OFFSET)) } + Self { + ids: raw_ids, + sentinel: MaybeUninit::zeroed(), + phantom: PhantomData, + } } } @@ -165,12 +166,6 @@ impl IdArray { pub trait IdTable { /// Obtain the pointer to the ID table. fn as_ptr(&self) -> *const T::RawType; - - /// Obtain the pointer to the bus specific device ID from an index. - fn id(&self, index: usize) -> &T::RawType; - - /// Obtain the pointer to the driver-specific information from an index. - fn info(&self, index: usize) -> &U; } impl IdTable for IdArray { @@ -179,28 +174,45 @@ fn as_ptr(&self) -> *const T::RawType { // to access the sentinel. core::ptr::from_ref(self).cast() } - - fn id(&self, index: usize) -> &T::RawType { - &self.raw_ids.ids[index] - } - - fn info(&self, index: usize) -> &U { - &self.id_infos[index] - } } /// Create device table alias for modpost. #[macro_export] macro_rules! module_device_table { - ($table_type: literal, $module_table_name:ident, $table_name:ident) => { - #[rustfmt::skip] + ( + $table_type: literal, $device_id_ty: ty, + $table_name: ident, $id_info_type: ty, + [$(($id: expr, $info:expr $(,)?)),* $(,)?] + ) => { #[export_name = - concat!("__mod_device_table__", line!(), + concat!("__mod_device_table__", ::core::line!(), "__kmod_", module_path!(), "__", $table_type, "__", stringify!($table_name)) ] - static $module_table_name: [::core::mem::MaybeUninit; $table_name.raw_ids().size()] = - unsafe { ::core::mem::transmute_copy($table_name.raw_ids()) }; + static $table_name: $crate::device_id::IdArray< + $device_id_ty, + $id_info_type, + { <[$device_id_ty]>::len(&[$($id,)*]) }, + > = $crate::device_id::IdArray::new([$(($id, &$info),)*]); + }; + + // Case for no ID info. + ( + $table_type: literal, $device_id_ty: ty, + $table_name: ident, @none, + [$($id: expr),* $(,)?] + ) => { + #[export_name = + concat!("__mod_device_table__", ::core::line!(), + "__kmod_", module_path!(), + "__", $table_type, + "__", stringify!($table_name)) + ] + static $table_name: $crate::device_id::IdArray< + $device_id_ty, + (), + { <[$device_id_ty]>::len(&[$($id,)*]) }, + > = $crate::device_id::IdArray::new_without_index([$($id),*]); }; } diff --git a/rust/kernel/devres.rs b/rust/kernel/devres.rs index 8ff8aedf251a..d2924aaae008 100644 --- a/rust/kernel/devres.rs +++ b/rust/kernel/devres.rs @@ -21,9 +21,12 @@ sync::{ aref::ARef, rcu, - Arc, // + Arc, + Completion, // }, types::{ + CovariantForLt, + ForLt, ForeignOwnable, Opaque, // }, @@ -37,6 +40,8 @@ struct Inner { node: Opaque, #[pin] data: Revocable, + #[pin] + revocation: Completion, } /// This abstraction is meant to be used by subsystems to containerize [`Device`] bound resources to @@ -53,6 +58,10 @@ struct Inner { /// After the [`Devres`] has been unbound it is not possible to access the encapsulated resource /// anymore. /// +/// When a [`Devres`] is dropped, it is guaranteed that `T` has been fully dropped by the time +/// [`Devres::drop`] returns, even if a concurrent revocation through the release callback is in +/// progress. +/// /// [`Devres`] users should make sure to simply free the corresponding backing resource in `T`'s /// [`Drop`] implementation. /// @@ -69,17 +78,19 @@ struct Inner { /// devres::Devres, /// io::{ /// Io, -/// IoKnownSize, +/// IoBase, /// Mmio, /// MmioRaw, -/// PhysAddr, // +/// MmioBackend, +/// PhysAddr, +/// Region, // /// }, /// prelude::*, /// }; /// use core::ops::Deref; /// /// // See also [`pci::Bar`] for a real example. -/// struct IoMem(MmioRaw); +/// struct IoMem(MmioRaw>); /// /// impl IoMem { /// /// # Safety @@ -94,7 +105,7 @@ struct Inner { /// return Err(ENOMEM); /// } /// -/// Ok(IoMem(MmioRaw::new(addr as usize, SIZE)?)) +/// Ok(IoMem(MmioRaw::new_region(addr as usize, SIZE)?)) /// } /// } /// @@ -105,12 +116,13 @@ struct Inner { /// } /// } /// -/// impl Deref for IoMem { -/// type Target = Mmio; +/// impl<'a, const SIZE: usize> IoBase<'a> for &'a IoMem { +/// type Backend = MmioBackend; +/// type Target = Region; /// -/// fn deref(&self) -> &Self::Target { +/// fn as_view(self) -> Mmio<'a, Region> { /// // SAFETY: The memory range stored in `self` has been properly mapped in `Self::new`. -/// unsafe { Mmio::from_raw(&self.0) } +/// unsafe { Mmio::from_raw(self.0) } /// } /// } /// # fn no_run(dev: &Device) -> Result<(), Error> { @@ -218,6 +230,7 @@ pub fn new(dev: &Device, data: impl PinInit) -> Result }; }), data <- Revocable::new(data), + revocation <- Completion::new(), }), GFP_KERNEL, )?; @@ -255,7 +268,14 @@ fn data(&self) -> &Revocable { // SAFETY: `inner` is a valid `Inner` pointer. let inner = unsafe { &*inner }; - inner.data.revoke(); + if inner.data.revoke() { + inner.revocation.complete_all(); + } else { + // Devres::drop() is concurrently revoking; wait for it to finish `drop_in_place()` + // before returning to `devres_release_all()`, ensuring `T` is fully torn down before + // the device finishes unbinding. + inner.revocation.wait_for_completion(); + } } #[allow(clippy::missing_safety_doc)] @@ -298,10 +318,7 @@ pub fn device(&self) -> &Device { /// use kernel::{ /// device::Core, /// devres::Devres, - /// io::{ - /// Io, - /// IoKnownSize, // - /// }, + /// io::Io, /// pci, // /// }; /// @@ -355,6 +372,8 @@ fn drop(&mut self) { // SAFETY: When `drop` runs, it is guaranteed that nobody is accessing the revocable data // anymore, hence it is safe not to wait for the grace period to finish. if unsafe { self.data().revoke_nosync() } { + self.inner.revocation.complete_all(); + // We revoked `self.data` before devres did, hence try to remove it. if self.remove_node() { // SAFETY: In `Self::new` we have taken an additional reference count of `self.data` @@ -362,10 +381,114 @@ fn drop(&mut self) { // this additional reference count. drop(unsafe { Arc::from_raw(Arc::as_ptr(&self.inner)) }); } + } else { + // The release callback is concurrently revoking; wait for it to finish + // `drop_in_place()` of the wrapped object before returning. + self.inner.revocation.wait_for_completion(); } } } +/// Guard returned by [`DevresLt::try_access`]. +/// +/// Dereferences to `F::Of<'a>`, shortening the lifetime of the stored data to the guard's borrow +/// lifetime. +pub struct DevresGuard<'a, F: CovariantForLt>(RevocableGuard<'a, F::Of<'static>>); + +impl<'a, F: CovariantForLt> core::ops::Deref for DevresGuard<'a, F> { + type Target = F::Of<'a>; + + #[inline] + fn deref(&self) -> &Self::Target { + F::cast_ref(&*self.0) + } +} + +/// Device-managed resource with [`ForLt`](trait@ForLt)-aware access. +/// +/// `DevresLt` wraps [`Devres`] and shortens the stored `'static` lifetime to the caller's borrow +/// lifetime in all access methods. +/// +/// Types that implement [`trait@CovariantForLt`] get direct-reference accessors ([`Self::access`], +/// [`Self::try_access`]). Plain [`ForLt`](trait@ForLt) types use closure-based accessors +/// ([`Self::access_with`], [`Self::try_access_with`]). +pub struct DevresLt(Devres>) +where + for<'a> F::Of<'a>: Send; + +impl DevresLt +where + for<'a> F::Of<'a>: Send, +{ + /// Creates a new [`DevresLt`] instance of the given `data`. + /// + /// # Safety + /// + /// The data must remain valid for the device's full bound scope. [`DevresLt`] allows + /// access until the device is unbound, which may outlast `'a`. + pub unsafe fn new<'a, E>( + dev: &'a Device, + data: impl PinInit, E>, + ) -> Result + where + Error: From, + { + // SAFETY: The caller guarantees the data is valid for the device's full bound scope. + // Lifetimes do not affect layout, so F::Of<'a> and F::Of<'static> have identical + // representation; casting the slot pointer is sound. + let data = unsafe { pin_init::cast_pin_init(data) }; + + Ok(Self(Devres::new(dev, data)?)) + } + + /// Return a reference of the [`Device`] this [`DevresLt`] instance has been created with. + #[inline] + pub fn device(&self) -> &Device { + self.0.device() + } + + /// Obtain `&F::Of<'_>`, bypassing the [`Revocable`], through a closure. + /// + /// This method works like [`DevresLt::access`](DevresLt::access) but accepts any + /// [`trait@ForLt`] type, not just [`trait@CovariantForLt`]. + #[inline] + pub fn access_with(&self, dev: &Device, f: G) -> Result + where + G: for<'a> FnOnce(&F::Of<'a>) -> R, + { + self.0.access(dev).map(f) + } + + /// [`DevresLt`] accessor for [`Revocable::try_access_with`]. + #[inline] + pub fn try_access_with(&self, f: G) -> Option + where + G: for<'a> FnOnce(&F::Of<'a>) -> R, + { + self.0.data().try_access_with(f) + } +} + +impl DevresLt +where + for<'a> F::Of<'a>: Send, +{ + /// Obtain `&'a F::Of<'a>`, bypassing the [`Revocable`]. + /// + /// This method works like [`Devres::access`], but shortens the returned reference's lifetime + /// from `'static` to `'a` via [`CovariantForLt::cast_ref`]. + #[inline] + pub fn access<'a>(&'a self, dev: &'a Device) -> Result<&'a F::Of<'a>> { + self.0.access(dev).map(F::cast_ref) + } + + /// [`DevresLt`] accessor for [`Revocable::try_access`]. + #[inline] + pub fn try_access(&self) -> Option> { + self.0.data().try_access().map(DevresGuard) + } +} + /// Consume `data` and [`Drop::drop`] `data` once `dev` is unbound. fn register_foreign

(dev: &Device, data: P) -> Result where diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 8e36a4e7f514..9a0bf745614c 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -14,14 +14,22 @@ }, error::to_result, fs::file, + io::{ + IoBackend, + IoBase, + IoCapable, + IoCopyable, + SysMem, + SysMemBackend, // + }, prelude::*, ptr::KnownSize, sync::aref::ARef, transmute::{ AsBytes, FromBytes, // - }, // - uaccess::UserSliceWriter, + }, + uaccess::UserSliceWriter, // }; use core::{ ops::{ @@ -654,52 +662,6 @@ pub unsafe fn as_mut(&self) -> &mut T { // SAFETY: per safety requirement. unsafe { &mut *self.as_mut_ptr() } } - - /// Reads the value of `field` and ensures that its type is [`FromBytes`]. - /// - /// # Safety - /// - /// This must be called from the [`dma_read`] macro which ensures that the `field` pointer is - /// validated beforehand. - /// - /// Public but hidden since it should only be used from [`dma_read`] macro. - #[doc(hidden)] - pub unsafe fn field_read(&self, field: *const F) -> F { - // SAFETY: - // - By the safety requirements field is valid. - // - Using read_volatile() here is not sound as per the usual rules, the usage here is - // a special exception with the following notes in place. When dealing with a potential - // race from a hardware or code outside kernel (e.g. user-space program), we need that - // read on a valid memory is not UB. Currently read_volatile() is used for this, and the - // rationale behind is that it should generate the same code as READ_ONCE() which the - // kernel already relies on to avoid UB on data races. Note that the usage of - // read_volatile() is limited to this particular case, it cannot be used to prevent - // the UB caused by racing between two kernel functions nor do they provide atomicity. - unsafe { field.read_volatile() } - } - - /// Writes a value to `field` and ensures that its type is [`AsBytes`]. - /// - /// # Safety - /// - /// This must be called from the [`dma_write`] macro which ensures that the `field` pointer is - /// validated beforehand. - /// - /// Public but hidden since it should only be used from [`dma_write`] macro. - #[doc(hidden)] - pub unsafe fn field_write(&self, field: *mut F, val: F) { - // SAFETY: - // - By the safety requirements field is valid. - // - Using write_volatile() here is not sound as per the usual rules, the usage here is - // a special exception with the following notes in place. When dealing with a potential - // race from a hardware or code outside kernel (e.g. user-space program), we need that - // write on a valid memory is not UB. Currently write_volatile() is used for this, and the - // rationale behind is that it should generate the same code as WRITE_ONCE() which the - // kernel already relies on to avoid UB on data races. Note that the usage of - // write_volatile() is limited to this particular case, it cannot be used to prevent - // the UB caused by racing between two kernel functions nor do they provide atomicity. - unsafe { field.write_volatile(val) } - } } impl Coherent { @@ -1005,7 +967,11 @@ fn write_to_slice( return Ok(0); }; - let count = self.size().saturating_sub(offset_val).min(writer.len()); + if offset_val >= self.size() { + return Ok(0); + } + + let count = (self.size() - offset_val).min(writer.len()); writer.write_dma(self, offset_val, count)?; @@ -1133,84 +1099,153 @@ unsafe impl Send for CoherentHandle {} // plain `Copy` values. unsafe impl Sync for CoherentHandle {} -/// Reads a field of an item from an allocated region of structs. +/// View type for `Coherent`. /// -/// The syntax is of the form `kernel::dma_read!(dma, proj)` where `dma` is an expression evaluating -/// to a [`Coherent`] and `proj` is a [projection specification](kernel::ptr::project!). -/// -/// # Examples -/// -/// ``` -/// use kernel::device::Device; -/// use kernel::dma::{attrs::*, Coherent}; -/// -/// struct MyStruct { field: u32, } -/// -/// // SAFETY: All bit patterns are acceptable values for `MyStruct`. -/// unsafe impl kernel::transmute::FromBytes for MyStruct{}; -/// // SAFETY: Instances of `MyStruct` have no uninitialized portions. -/// unsafe impl kernel::transmute::AsBytes for MyStruct{}; -/// -/// # fn test(alloc: &kernel::dma::Coherent<[MyStruct]>) -> Result { -/// let whole = kernel::dma_read!(alloc, [try: 2]); -/// let field = kernel::dma_read!(alloc, [panic: 1].field); -/// # Ok::<(), Error>(()) } -/// ``` -#[macro_export] -macro_rules! dma_read { - ($dma:expr, $($proj:tt)*) => {{ - let dma = &$dma; - let ptr = $crate::ptr::project!( - $crate::dma::Coherent::as_ptr(dma), $($proj)* - ); - // SAFETY: The pointer created by the projection is within the DMA region. - unsafe { $crate::dma::Coherent::field_read(dma, ptr) } - }}; +/// This is same as [`SysMem`] but with additional information that allows handing out a DMA handle. +pub struct CoherentView<'a, T: ?Sized> { + cpu_addr: SysMem<'a, T>, + dma_handle: DmaAddress, } -/// Writes to a field of an item from an allocated region of structs. -/// -/// The syntax is of the form `kernel::dma_write!(dma, proj, val)` where `dma` is an expression -/// evaluating to a [`Coherent`], `proj` is a -/// [projection specification](kernel::ptr::project!), and `val` is the value to be written to the -/// projected location. -/// -/// # Examples -/// -/// ``` -/// use kernel::device::Device; -/// use kernel::dma::{attrs::*, Coherent}; -/// -/// struct MyStruct { member: u32, } -/// -/// // SAFETY: All bit patterns are acceptable values for `MyStruct`. -/// unsafe impl kernel::transmute::FromBytes for MyStruct{}; -/// // SAFETY: Instances of `MyStruct` have no uninitialized portions. -/// unsafe impl kernel::transmute::AsBytes for MyStruct{}; -/// -/// # fn test(alloc: &kernel::dma::Coherent<[MyStruct]>) -> Result { -/// kernel::dma_write!(alloc, [try: 2].member, 0xf); -/// kernel::dma_write!(alloc, [panic: 1], MyStruct { member: 0xf }); -/// # Ok::<(), Error>(()) } -/// ``` -#[macro_export] -macro_rules! dma_write { - (@parse [$dma:expr] [$($proj:tt)*] [, $val:expr]) => {{ - let dma = &$dma; - let ptr = $crate::ptr::project!( - mut $crate::dma::Coherent::as_mut_ptr(dma), $($proj)* - ); - let val = $val; - // SAFETY: The pointer created by the projection is within the DMA region. - unsafe { $crate::dma::Coherent::field_write(dma, ptr, val) } - }}; - (@parse [$dma:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => { - $crate::dma_write!(@parse [$dma] [$($proj)* .$field] [$($rest)*]) - }; - (@parse [$dma:expr] [$($proj:tt)*] [[$flavor:ident: $index:expr] $($rest:tt)*]) => { - $crate::dma_write!(@parse [$dma] [$($proj)* [$flavor: $index]] [$($rest)*]) - }; - ($dma:expr, $($rest:tt)*) => { - $crate::dma_write!(@parse [$dma] [] [$($rest)*]) - }; +impl Copy for CoherentView<'_, T> {} +impl Clone for CoherentView<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl<'a, T: ?Sized> CoherentView<'a, T> { + /// Erase the DMA handle information and obtain a [`SysMem`] view of the same memory region. + #[inline] + pub fn as_sys_mem(self) -> SysMem<'a, T> { + self.cpu_addr + } + + /// Returns a DMA handle which may be given to the device as the DMA address base of the region. + #[inline] + pub fn dma_handle(self) -> DmaAddress { + self.dma_handle + } + + /// Returns a reference to the data in the region. + /// + /// # Safety + /// + /// * Callers must ensure that the device does not read/write to/from memory while the returned + /// reference is live. + /// * Callers must ensure that this call does not race with a write (including call to `as_mut`) + /// to the same region while the returned reference is live. + #[inline] + pub unsafe fn as_ref(self) -> &'a T { + // SAFETY: pointer is aligned and valid per type invariant. Aliasing rule is satisfied per + // safety requirement. + unsafe { &*self.cpu_addr.as_ptr() } + } + + /// Returns a mutable reference to the data in the region. + /// + /// # Safety + /// + /// * Callers must ensure that the device does not read/write to/from memory while the returned + /// reference is live. + /// * Callers must ensure that this call does not race with a read (including call to `as_ref`) + /// or write (including call to `as_mut`) to the same region while the returned reference is + /// live. + #[inline] + pub unsafe fn as_mut(self) -> &'a mut T { + // SAFETY: pointer is aligned and valid per type invariant. Aliasing rule is satisfied per + // safety requirement. + unsafe { &mut *self.cpu_addr.as_ptr() } + } +} + +/// `IoBackend` implementation for `Coherent`. +pub struct CoherentIoBackend; + +impl IoBackend for CoherentIoBackend { + type View<'a, T: ?Sized + KnownSize> = CoherentView<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + SysMemBackend::as_ptr(view.cpu_addr) + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + let offset = ptr.addr() - view.cpu_addr.as_ptr().addr(); + // CAST: The offset DMA address can never overflow. + let dma_handle = view.dma_handle + offset as DmaAddress; + CoherentView { + dma_handle, + // SAFETY: Per safety requirement. + cpu_addr: unsafe { SysMemBackend::project_view(view.cpu_addr, ptr) }, + } + } +} + +impl IoCapable for CoherentIoBackend +where + SysMemBackend: IoCapable, +{ + #[inline] + fn io_read<'a>(view: Self::View<'a, T>) -> T { + SysMemBackend::io_read(view.cpu_addr) + } + + #[inline] + fn io_write<'a>(view: Self::View<'a, T>, value: T) { + SysMemBackend::io_write(view.cpu_addr, value) + } +} + +impl IoCopyable for CoherentIoBackend { + #[inline] + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + // SAFETY: Per safety requirement. + unsafe { SysMemBackend::copy_from_io(view.cpu_addr, buffer) } + } + + #[inline] + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + // SAFETY: Per safety requirement. + unsafe { SysMemBackend::copy_to_io(view.cpu_addr, buffer) } + } + + #[inline] + fn copy_read(view: Self::View<'_, T>) -> T { + SysMemBackend::copy_read(view.cpu_addr) + } + + #[inline] + fn copy_write(view: Self::View<'_, T>, value: T) { + SysMemBackend::copy_write(view.cpu_addr, value) + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for CoherentView<'a, T> { + type Backend = CoherentIoBackend; + type Target = T; + + #[inline] + fn as_view(self) -> CoherentView<'a, Self::Target> { + self + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for &'a Coherent { + type Backend = CoherentIoBackend; + type Target = T; + + #[inline] + fn as_view(self) -> CoherentView<'a, Self::Target> { + CoherentView { + // SAFETY: `cpu_addr` is valid and aligned kernel accessible memory. + cpu_addr: unsafe { SysMem::new(self.cpu_addr.as_ptr()) }, + dma_handle: self.dma_handle, + } + } } diff --git a/rust/kernel/driver.rs b/rust/kernel/driver.rs index bf5ba0d27553..c9c74c4dde8f 100644 --- a/rust/kernel/driver.rs +++ b/rust/kernel/driver.rs @@ -291,26 +291,6 @@ fn init( } } -// Calling the FFI function directly from the `Adapter` impl may result in it being called -// directly from driver modules. This happens since the Rust compiler will use monomorphisation, so -// it might happen that functions are instantiated within the calling driver module. For now, work -// around this with `#[inline(never)]` helpers. -// -// TODO: Remove once a more generic solution has been implemented. For instance, we may be able to -// leverage `bindgen` to take care of this depending on whether a symbol is (already) exported. -#[inline(never)] -#[allow(clippy::missing_safety_doc)] -#[allow(dead_code)] -#[must_use] -unsafe fn acpi_of_match_device( - adev: *const bindings::acpi_device, - of_match_table: *const bindings::of_device_id, - of_id: *mut *const bindings::of_device_id, -) -> bool { - // SAFETY: Safety requirements are the same as `bindings::acpi_of_match_device`. - unsafe { bindings::acpi_of_match_device(adev, of_match_table, of_id) } -} - /// The bus independent adapter to match a drivers and a devices. /// /// This trait should be implemented by the bus specific adapter, which represents the connection @@ -324,118 +304,23 @@ pub trait Adapter { /// The [`acpi::IdTable`] of the corresponding driver fn acpi_id_table() -> Option>; - /// Returns the driver's private data from the matching entry in the [`acpi::IdTable`], if any. - /// - /// If this returns `None`, it means there is no match with an entry in the [`acpi::IdTable`]. - fn acpi_id_info(dev: &device::Device) -> Option<&'static Self::IdInfo> { - #[cfg(not(CONFIG_ACPI))] - { - let _ = dev; - None - } - - #[cfg(CONFIG_ACPI)] - { - let table = Self::acpi_id_table()?; - - // SAFETY: - // - `table` has static lifetime, hence it's valid for read, - // - `dev` is guaranteed to be valid while it's alive, and so is `dev.as_raw()`. - let raw_id = unsafe { bindings::acpi_match_device(table.as_ptr(), dev.as_raw()) }; - - if raw_id.is_null() { - None - } else { - // SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct acpi_device_id` - // and does not add additional invariants, so it's safe to transmute. - let id = unsafe { &*raw_id.cast::() }; - - Some(table.info(::index(id))) - } - } - } - /// The [`of::IdTable`] of the corresponding driver. fn of_id_table() -> Option>; - /// Returns the driver's private data from the matching entry in the [`of::IdTable`], if any. - /// - /// If this returns `None`, it means there is no match with an entry in the [`of::IdTable`]. - fn of_id_info(dev: &device::Device) -> Option<&'static Self::IdInfo> { - let table = Self::of_id_table()?; - - #[cfg(not(any(CONFIG_OF, CONFIG_ACPI)))] - { - let _ = (dev, table); - } - - #[cfg(CONFIG_OF)] - { - // SAFETY: - // - `table` has static lifetime, hence it's valid for read, - // - `dev` is guaranteed to be valid while it's alive, and so is `dev.as_raw()`. - let raw_id = unsafe { bindings::of_match_device(table.as_ptr(), dev.as_raw()) }; - - if !raw_id.is_null() { - // SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct of_device_id` - // and does not add additional invariants, so it's safe to transmute. - let id = unsafe { &*raw_id.cast::() }; - - return Some(table.info( - ::index(id), - )); - } - } - - #[cfg(CONFIG_ACPI)] - { - use core::ptr; - use device::property::FwNode; - - let mut raw_id = ptr::null(); - - let fwnode = dev.fwnode().map_or(ptr::null_mut(), FwNode::as_raw); - - // SAFETY: `fwnode` is a pointer to a valid `fwnode_handle`. A null pointer will be - // passed through the function. - let adev = unsafe { bindings::to_acpi_device_node(fwnode) }; - - // SAFETY: - // - `adev` is a valid pointer to `acpi_device` or is null. It is guaranteed to be - // valid as long as `dev` is alive. - // - `table` has static lifetime, hence it's valid for read. - if unsafe { acpi_of_match_device(adev, table.as_ptr(), &raw mut raw_id) } { - // SAFETY: - // - the function returns true, therefore `raw_id` has been set to a pointer to a - // valid `of_device_id`. - // - `DeviceId` is a `#[repr(transparent)]` wrapper of `struct of_device_id` - // and does not add additional invariants, so it's safe to transmute. - let id = unsafe { &*raw_id.cast::() }; - - return Some(table.info( - ::index(id), - )); - } - } - - None - } - /// Returns the driver's private data from the matching entry of any of the ID tables, if any. /// /// If this returns `None`, it means that there is no match in any of the ID tables directly /// associated with a [`device::Device`]. - fn id_info(dev: &device::Device) -> Option<&'static Self::IdInfo> { - let id = Self::acpi_id_info(dev); - if id.is_some() { - return id; - } + /// + /// # Safety + /// + /// The caller must ensure that the `dev` matched data is of type `Self::IdInfo`. + #[inline] + unsafe fn id_info(dev: &device::Device) -> Option<&'static Self::IdInfo> { + // SAFETY: `dev` is guaranteed to be valid while it's alive, and so is `dev.as_raw()`. + let data = unsafe { bindings::device_get_match_data(dev.as_raw()) }; - let id = Self::of_id_info(dev); - if id.is_some() { - return id; - } - - None + // SAFETY: Per safety requirement, `data` is of type `Self::IdInfo`. + unsafe { data.cast::().as_ref() } } } diff --git a/rust/kernel/firmware.rs b/rust/kernel/firmware.rs index 71168d8004e2..1e3dd2e3422d 100644 --- a/rust/kernel/firmware.rs +++ b/rust/kernel/firmware.rs @@ -51,12 +51,8 @@ fn request_nowarn() -> Self { /// # Examples /// /// ```no_run -/// # use kernel::{device::Device, firmware::Firmware}; -/// -/// # fn no_run() -> Result<(), Error> { -/// # // SAFETY: *NOT* safe, just for the example to get an `ARef` instance -/// # let dev = unsafe { Device::get_device(core::ptr::null_mut()) }; -/// +/// # use kernel::{device::Device, firmware::Firmware, sync::aref::ARef}; +/// # fn no_run(dev: ARef) -> Result<(), Error> { /// let fw = Firmware::request(c"path/to/firmware.bin", &dev)?; /// let blob = fw.data(); /// diff --git a/rust/kernel/i2c.rs b/rust/kernel/i2c.rs index dd9271af5eb8..fbc5fb06ca16 100644 --- a/rust/kernel/i2c.rs +++ b/rust/kernel/i2c.rs @@ -65,10 +65,6 @@ unsafe impl RawDeviceId for DeviceId { // SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. unsafe impl RawDeviceIdIndex for DeviceId { const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::i2c_device_id, driver_data); - - fn index(&self) -> usize { - self.0.driver_data - } } /// IdTable type for I2C @@ -77,14 +73,8 @@ fn index(&self) -> usize { /// Create a I2C `IdTable` with its alias for modpost. #[macro_export] macro_rules! i2c_device_table { - ($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => { - const $table_name: $crate::device_id::IdArray< - $crate::i2c::DeviceId, - $id_info_type, - { $table_data.len() }, - > = $crate::device_id::IdArray::new($table_data); - - $crate::module_device_table!("i2c", $module_table_name, $table_name); + ($($tt:tt)*) => { + $crate::module_device_table!("i2c", $crate::i2c::DeviceId, $($tt)*); }; } @@ -159,8 +149,10 @@ extern "C" fn probe_callback(idev: *mut bindings::i2c_client) -> kernel::ffi::c_ // INVARIANT: `idev` is valid for the duration of `probe_callback()`. let idev = unsafe { &*idev.cast::>>() }; - let info = - Self::i2c_id_info(idev).or_else(|| ::id_info(idev.as_ref())); + let info = Self::i2c_id_info(idev).or_else(|| { + // SAFETY: `idev` matched data is of type `Self::IdInfo`. + unsafe { ::id_info(idev.as_ref()) } + }); from_result(|| { let data = T::probe(idev, info); @@ -218,7 +210,8 @@ fn i2c_id_info(dev: &I2cClient) -> Option<&'static ::Id // does not add additional invariants, so it's safe to transmute. let id = unsafe { &*raw_id.cast::() }; - Some(table.info(::index(id))) + // SAFETY: `id` comes from `table` which is of type `IdArray<_, Self::IdInfo>`. + Some(unsafe { id.info_unchecked::() }) } } @@ -267,7 +260,6 @@ macro_rules! module_i2c_driver { /// /// kernel::acpi_device_table!( /// ACPI_TABLE, -/// MODULE_ACPI_TABLE, /// ::IdInfo, /// [ /// (acpi::DeviceId::new(c"LNUXBEEF"), ()) @@ -276,7 +268,6 @@ macro_rules! module_i2c_driver { /// /// kernel::i2c_device_table!( /// I2C_TABLE, -/// MODULE_I2C_TABLE, /// ::IdInfo, /// [ /// (i2c::DeviceId::new(c"rust_driver_i2c"), ()) @@ -285,7 +276,6 @@ macro_rules! module_i2c_driver { /// /// kernel::of_device_table!( /// OF_TABLE, -/// MODULE_OF_TABLE, /// ::IdInfo, /// [ /// (of::DeviceId::new(c"test,device"), ()) diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index d4063ee41200..5ce9fd129068 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -4,9 +4,18 @@ //! //! C header: [`include/asm-generic/io.h`](srctree/include/asm-generic/io.h) +use core::{ + marker::PhantomData, + mem::MaybeUninit, // +}; + use crate::{ bindings, - prelude::*, // + prelude::*, + ptr::{ + Alignment, + KnownSize, // + }, // }; #[cfg(CONFIG_HAS_IOMEM)] @@ -32,129 +41,226 @@ /// `CONFIG_PHYS_ADDR_T_64BIT`, and it can be a u64 even on 32-bit architectures. pub type ResourceSize = bindings::resource_size_t; +/// Untyped I/O region. +/// +/// This type can be used when an I/O region without known type information has a compile-time known +/// minimum size (and a runtime known actual size). +/// +/// # Invariants +/// +/// - Size of the region is at least as large as the `SIZE` generic parameter. +/// - Size of the region is multiple of 4. +#[repr(C, align(4))] +#[derive(FromBytes)] +pub struct Region { + inner: [u8], +} + +impl Region { + /// Create a raw mutable pointer from given base address and size. + /// + /// `size` should be at least as large as the minimum size `SIZE`, and `base` and `size` should + /// be 4-byte aligned to uphold the type invariant. + /// + /// Just like other methods on raw pointers, it is not unsafe to create a raw pointer + /// that does not uphold the type invariants. However such pointers are not valid. + #[inline] + pub fn ptr_from_raw_parts_mut(base: *mut u8, size: usize) -> *mut Self { + core::ptr::slice_from_raw_parts_mut(base, size) as *mut Region + } + + /// Create a raw mutable pointer from given base address and size. + /// + /// The alignment of `base` is checked, and `size` is checked against the minimum size specified + /// via const generics. + #[inline] + pub fn ptr_try_from_raw_parts_mut(base: *mut u8, size: usize) -> Result<*mut Self> { + if size < SIZE || base.align_offset(4) != 0 || !size.is_multiple_of(4) { + return Err(EINVAL); + } + + Ok(Self::ptr_from_raw_parts_mut(base, size)) + } +} + +impl KnownSize for Region { + const MIN_SIZE: usize = SIZE; + // Alignment of 4 is the most common; different base types can be added once required. + const MIN_ALIGN: Alignment = Alignment::new::<4>(); + + #[inline(always)] + fn size(p: *const Self) -> usize { + (p as *const [u8]).len() + } +} + +// SAFETY: +// - Values read from I/O are always treated as initialized. +// - Per type invariant the size is multiple of 4 and the type is 4-byte aligned, so it is padding +// free. +// +// This cannot be derived as `derive(IntoBytes)` as the padding free property comes from type +// invariant which the macro does not know. +unsafe impl IntoBytes for Region { + #[inline] + #[allow(unused)] // Rust 1.87+ stops requiring this and will emit unused warnings. + fn only_derive_is_allowed_to_implement_this_trait() {} +} + /// Raw representation of an MMIO region. /// +/// `MmioRaw` is equivalent to `T __iomem *` in C. +/// /// By itself, the existence of an instance of this structure does not provide any guarantees that /// the represented MMIO region does exist or is properly mapped. /// /// Instead, the bus specific MMIO implementation must convert this raw representation into an /// `Mmio` instance providing the actual memory accessors. Only by the conversion into an `Mmio` /// structure any guarantees are given. -pub struct MmioRaw { - addr: usize, - maxsize: usize, +pub struct MmioRaw { + /// Pointer is in I/O address space. + /// + /// The provenance does not matter, only the address and metadata do. + ptr: *mut T, } -impl MmioRaw { - /// Returns a new `MmioRaw` instance on success, an error otherwise. - pub fn new(addr: usize, maxsize: usize) -> Result { - if maxsize < SIZE { - return Err(EINVAL); - } - - Ok(Self { addr, maxsize }) +impl Copy for MmioRaw {} +impl Clone for MmioRaw { + #[inline] + fn clone(&self) -> Self { + *self } +} +// SAFETY: `MmioRaw` is just an address, so is thread-safe. +unsafe impl Send for MmioRaw {} +// SAFETY: `MmioRaw` is just an address, so is thread-safe. +unsafe impl Sync for MmioRaw {} + +impl MmioRaw { + /// Create a `MmioRaw` from address. + #[inline] + pub fn new(addr: usize) -> Self { + Self { + ptr: core::ptr::without_provenance_mut(addr), + } + } +} + +impl MmioRaw> { + /// Create a `MmioRaw` representing a I/O region with given size. + /// + /// The size is checked against the minimum size specified via const generics. + #[inline] + pub fn new_region(addr: usize, size: usize) -> Result { + Ok(Self { + ptr: Region::ptr_try_from_raw_parts_mut(core::ptr::without_provenance_mut(addr), size)?, + }) + } +} + +impl MmioRaw { /// Returns the base address of the MMIO region. #[inline] pub fn addr(&self) -> usize { - self.addr + self.ptr.addr() } - /// Returns the maximum size of the MMIO region. + /// Returns the size of the MMIO region. #[inline] - pub fn maxsize(&self) -> usize { - self.maxsize + pub fn size(&self) -> usize { + KnownSize::size(self.ptr) } } -/// IO-mapped memory region. -/// -/// The creator (usually a subsystem / bus such as PCI) is responsible for creating the -/// mapping, performing an additional region request etc. -/// -/// # Invariant -/// -/// `addr` is the start and `maxsize` the length of valid I/O mapped memory region of size -/// `maxsize`. -/// -/// # Examples -/// -/// ```no_run -/// # #![cfg(CONFIG_HAS_IOMEM)] -/// use kernel::{ -/// bindings, -/// ffi::c_void, -/// io::{ -/// Io, -/// IoKnownSize, -/// Mmio, -/// MmioRaw, -/// PhysAddr, -/// }, -/// }; -/// use core::ops::Deref; -/// -/// // See also `pci::Bar` for a real example. -/// struct IoMem(MmioRaw); -/// -/// impl IoMem { -/// /// # Safety -/// /// -/// /// [`paddr`, `paddr` + `SIZE`) must be a valid MMIO region that is mappable into the CPUs -/// /// virtual address space. -/// unsafe fn new(paddr: usize) -> Result{ -/// // SAFETY: By the safety requirements of this function [`paddr`, `paddr` + `SIZE`) is -/// // valid for `ioremap`. -/// let addr = unsafe { bindings::ioremap(paddr as PhysAddr, SIZE) }; -/// if addr.is_null() { -/// return Err(ENOMEM); -/// } -/// -/// Ok(IoMem(MmioRaw::new(addr as usize, SIZE)?)) -/// } -/// } -/// -/// impl Drop for IoMem { -/// fn drop(&mut self) { -/// // SAFETY: `self.0.addr()` is guaranteed to be properly mapped by `Self::new`. -/// unsafe { bindings::iounmap(self.0.addr() as *mut c_void); }; -/// } -/// } -/// -/// impl Deref for IoMem { -/// type Target = Mmio; -/// -/// fn deref(&self) -> &Self::Target { -/// // SAFETY: The memory range stored in `self` has been properly mapped in `Self::new`. -/// unsafe { Mmio::from_raw(&self.0) } -/// } -/// } -/// -///# fn no_run() -> Result<(), Error> { -/// // SAFETY: Invalid usage for example purposes. -/// let iomem = unsafe { IoMem::<{ core::mem::size_of::() }>::new(0xBAAAAAAD)? }; -/// iomem.write32(0x42, 0x0); -/// assert!(iomem.try_write32(0x42, 0x0).is_ok()); -/// assert!(iomem.try_write32(0x42, 0x4).is_err()); -/// # Ok(()) -/// # } -/// ``` -#[repr(transparent)] -pub struct Mmio(MmioRaw); - -/// Checks whether an access of type `U` at the given `offset` +/// Checks whether an access of type `U` at the given `base` and the given `offset` /// is valid within this region. +/// +/// The `base` is used for alignment checking only. This can be set to 0 to skip the check. #[inline] -const fn offset_valid(offset: usize, size: usize) -> bool { - let type_size = core::mem::size_of::(); - if let Some(end) = offset.checked_add(type_size) { - end <= size && offset % type_size == 0 +const fn offset_valid(base: usize, offset: usize, size: usize) -> bool { + if let Some(end) = offset.checked_add(size_of::()) { + end <= size && (base.wrapping_add(offset) % align_of::() == 0) } else { false } } +/// Returns a view for a given `offset`, performing compile-time bound checks. +// Always inline to optimize out error path of `build_assert`. +#[inline(always)] +fn io_view_assert<'a, IO: Io<'a>, U>( + this: IO, + offset: usize, +) -> ::View<'a, U> { + // We cannot check alignment with `offset_valid` using `ptr.addr()`. So set 0 for it and + // ensure alignment by checking that the alignment of `U` is smaller or equal to the + // alignment of `IO::Target`. + const_assert!(Alignment::of::().as_usize() <= IO::Target::MIN_ALIGN.as_usize()); + build_assert!(offset_valid::(0, offset, IO::Target::MIN_SIZE)); + + let view = this.as_view(); + let ptr = IO::Backend::as_ptr(view); + let projected_ptr = ptr.cast::().wrapping_byte_add(offset); + // SAFETY: `offset_valid` checks for size and alignment and therefore `projected_ptr` is a + // valid projection. + unsafe { IO::Backend::project_view(view, projected_ptr) } +} + +/// Returns a view for a given `offset`, performing runtime bound checks. +#[inline] +fn io_view<'a, IO: Io<'a>, U>( + this: IO, + offset: usize, +) -> Result<::View<'a, U>> { + let view = this.as_view(); + let ptr = IO::Backend::as_ptr(view); + + if !offset_valid::(ptr.addr(), offset, KnownSize::size(ptr)) { + return Err(EINVAL); + } + + let projected_ptr = ptr.cast::().wrapping_byte_add(offset); + // SAFETY: `offset_valid` checks for size and alignment and therefore `projected_ptr` is a + // valid projection. + Ok(unsafe { IO::Backend::project_view(view, projected_ptr) }) +} + +/// I/O backends. +/// +/// This is an abstract representation to be implemented by arbitrary I/O +/// backends (e.g. MMIO, PCI config space, etc.). +/// +/// The base trait only defines the projection operations; which I/O methods are available depends +/// on which [`IoCapable`] traits are implemented for the type. For example, for MMIO regions, +/// all widths (u8, u16, u32, and u64 on 64-bit systems) are typically supported. For PCI +/// configuration space, u8, u16, and u32 are supported but u64 is not. +/// +/// This trait is separate from the `Io` trait as multiple different I/O types may share the same +/// operation. +pub trait IoBackend { + /// View type for this I/O backend. + type View<'a, T: ?Sized + KnownSize>: IoBase<'a, Backend = Self, Target = T>; + + /// Convert a `view` to a raw pointer for projection. + /// + /// The returned pointer is private implementation detail of the backend; it is likely not + /// valid. It should not be dereferenced. + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T; + + /// Project `view` to its subregion indicated by `ptr`. + /// + /// If input `view` is valid, returned view must also be valid. + /// + /// # Safety + /// + /// `ptr` must be a projection of `Self::as_ptr(view)`. + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U>; +} + /// Trait indicating that an I/O backend supports operations of a certain type and providing an /// implementation for these operations. /// @@ -163,20 +269,75 @@ const fn offset_valid(offset: usize, size: usize) -> bool { /// For example, a PCI configuration space may implement `IoCapable`, `IoCapable`, /// and `IoCapable`, but not `IoCapable`, while an MMIO region on a 64-bit /// system might implement all four. -pub trait IoCapable { - /// Performs an I/O read of type `T` at `address` and returns the result. - /// - /// # Safety - /// - /// The range `[address..address + size_of::()]` must be within the bounds of `Self`. - unsafe fn io_read(&self, address: usize) -> T; +pub trait IoCapable: IoBackend { + /// Performs an I/O read of type `T` at `view` and returns the result. + fn io_read<'a>(view: Self::View<'a, T>) -> T; - /// Performs an I/O write of `value` at `address`. + /// Performs an I/O write of `value` at `view`. + fn io_write<'a>(view: Self::View<'a, T>, value: T); +} + +/// Trait indicating that an I/O backend supports memory copy operations. +pub trait IoCopyable: IoBackend { + /// Copy contents of `view` to `buffer`. /// /// # Safety /// - /// The range `[address..address + size_of::()]` must be within the bounds of `Self`. - unsafe fn io_write(&self, value: T, address: usize); + /// - `buffer` is valid for volatile write for `view.size()` bytes. + /// - `buffer` should not overlap with `view`. + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8); + + /// Copy contents from `buffer` to `view`. + /// + /// # Safety + /// + /// - `buffer` is valid for volatile read for `view.size()` bytes. + /// - `buffer` should not overlap with `view`. + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8); + + /// Copy from `view` and return the value. + #[inline] + fn copy_read(view: Self::View<'_, T>) -> T { + // Project `self` to `[u8]`. + let ptr = Self::as_ptr(view); + // SAFETY: This is a identity projection. + let slice_view = unsafe { + Self::project_view( + view, + core::ptr::slice_from_raw_parts_mut::(ptr.cast(), size_of::()), + ) + }; + + let mut buf = MaybeUninit::::uninit(); + // SAFETY: + // - `buf.as_mut_ptr()` is valid for write for `size_of::()` bytes. + // - `buf` is local so `buf.as_mut_ptr()` cannot overlap with `slice_view`. + unsafe { Self::copy_from_io(slice_view, buf.as_mut_ptr().cast()) }; + // SAFETY: `T: FromBytes` guarantee that all bit patterns are valid. + unsafe { buf.assume_init() } + } + + /// Copy `value` to `view`. + /// + /// Destructor of `value` will not be executed, consistent with [`zerocopy::transmute`]. + #[inline] + fn copy_write(view: Self::View<'_, T>, value: T) { + // Project `self` to `[u8]`. + let ptr = Self::as_ptr(view); + // SAFETY: This is a identity projection. + let slice_view = unsafe { + Self::project_view( + view, + core::ptr::slice_from_raw_parts_mut::(ptr.cast(), size_of::()), + ) + }; + + // SAFETY: + // - `&raw const value` is valid for read for `size_of::()` bytes. + // - `value` is local so `&raw const value` cannot overlap with `slice_view`. + unsafe { Self::copy_to_io(slice_view, (&raw const value).cast()) }; + core::mem::forget(value); + } } /// Describes a given I/O location: its offset, width, and type to convert the raw value from and @@ -188,15 +349,16 @@ pub trait IoCapable { /// (for primitive types like [`u32`]) and typed ones (like those generated by the [`register!`] /// macro). /// -/// An `IoLoc` carries three pieces of information: +/// An `IoLoc` carries the following pieces of information: /// +/// - The valid `Base` to operate on. For most registers, this should be [`Region`]. /// - The offset to access (returned by [`IoLoc::offset`]), /// - The width of the access (determined by [`IoLoc::IoType`]), /// - The type `T` in which the raw data is returned or provided. /// /// `T` and `IoLoc::IoType` may differ: for instance, a typed register has `T` = the register type /// with its bitfields, and `IoType` = its backing primitive (e.g. `u32`). -pub trait IoLoc { +pub trait IoLoc { /// Size ([`u8`], [`u16`], etc) of the I/O performed on the returned [`offset`](IoLoc::offset). type IoType: Into + From; @@ -204,12 +366,12 @@ pub trait IoLoc { fn offset(self) -> usize; } -/// Implements [`IoLoc<$ty>`] for [`usize`], allowing [`usize`] to be used as a parameter of -/// [`Io::read`] and [`Io::write`]. +/// Implements [`IoLoc, $ty>`] for [`usize`], allowing [`usize`] to be used as a +/// parameter of [`Io::read`] and [`Io::write`]. macro_rules! impl_usize_ioloc { ($($ty:ty),*) => { $( - impl IoLoc<$ty> for usize { + impl IoLoc, $ty> for usize { type IoType = $ty; #[inline(always)] @@ -227,181 +389,430 @@ fn offset(self) -> usize { /// Types implementing this trait (e.g. MMIO BARs or PCI config regions) /// can perform I/O operations on regions of memory. /// -/// This is an abstract representation to be implemented by arbitrary I/O -/// backends (e.g. MMIO, PCI config space, etc.). +/// This trait defines which backend shall be used for I/O operations and provides a method to +/// convert into [`IoBackend::View`]. Users should use the [`Io`] trait which provides the actual +/// methods to perform I/O operations. /// -/// The [`Io`] trait provides: -/// - Base address and size information +/// This should be implemented on cheaply copyable handles, such as references or view types. +pub trait IoBase<'a>: Copy { + /// Type that defines all I/O operations. + type Backend: IoBackend; + + /// Type of this I/O region. For untyped regions, [`Region`] can be used. + type Target: ?Sized + KnownSize; + + /// Return a view that covers the full region. + fn as_view(self) -> ::View<'a, Self::Target>; +} + +/// Extension trait to provide I/O operation methods to types that implement [`IoBase`]. +/// +/// This trait provides: /// - Helper methods for offset validation and address calculation /// - Fallible (runtime checked) accessors for different data widths /// -/// Which I/O methods are available depends on which [`IoCapable`] traits -/// are implemented for the type. -/// -/// # Examples -/// -/// For MMIO regions, all widths (u8, u16, u32, and u64 on 64-bit systems) are typically -/// supported. For PCI configuration space, u8, u16, and u32 are supported but u64 is not. -pub trait Io { - /// Returns the base address of this mapping. - fn addr(&self) -> usize; - - /// Returns the maximum size of this mapping. - fn maxsize(&self) -> usize; - - /// Returns the absolute I/O address for a given `offset`, - /// performing runtime bound checks. +/// Which I/O methods are available depends on the associated [`IoBackend`] implementation. +pub trait Io<'a>: IoBase<'a> { + /// Returns the size of this I/O region. #[inline] - fn io_addr(&self, offset: usize) -> Result { - if !offset_valid::(offset, self.maxsize()) { + fn size(self) -> usize { + KnownSize::size(Self::Backend::as_ptr(self.as_view())) + } + + /// Returns the length of the slice in number of elements. + #[inline] + fn len(self) -> usize + where + Self: Io<'a, Target = [T]>, + { + Self::Backend::as_ptr(self.as_view()).len() + } + + /// Returns `true` if the slice has a length of 0. + #[inline] + fn is_empty(self) -> bool + where + Self: Io<'a, Target = [T]>, + { + self.len() == 0 + } + + /// Try to convert into a different typed I/O view. + /// + /// A runtime check is performed to ensure that the target type is of same or smaller size to + /// current type, and the current view is properly aligned for the target type. Returns + /// `Err(EINVAL)` if the runtime check fails. + /// + /// # Examples + /// + /// ```no_run + /// use kernel::io::{ + /// io_project, + /// Mmio, + /// Io, + /// Region, + /// }; + /// #[derive(FromBytes, IntoBytes)] + /// #[repr(C)] + /// struct MyStruct { field: u32, } + /// + /// # fn test(mmio: &Mmio<'_, Region>) -> Result { + /// // let mmio: Mmio<'_, Region>; + /// let whole: Mmio<'_, MyStruct> = mmio.try_cast()?; + /// # Ok::<(), Error>(()) } + /// ``` + #[inline] + fn try_cast(self) -> Result<::View<'a, U>> + where + Self::Target: FromBytes + IntoBytes, + U: FromBytes + IntoBytes, + { + let view = self.as_view(); + let ptr = Self::Backend::as_ptr(view); + + if size_of::() > KnownSize::size(ptr) { return Err(EINVAL); } - // Probably no need to check, since the safety requirements of `Self::new` guarantee that - // this can't overflow. - self.addr().checked_add(offset).ok_or(EINVAL) + if ptr.addr() % align_of::() != 0 { + return Err(EINVAL); + } + + // SAFETY: We have checked bounds and alignment, so this is a valid projection. + Ok(unsafe { Self::Backend::project_view(view, ptr.cast()) }) + } + + /// Read a value from I/O. + /// + /// This only works for primitives supported by the I/O backend. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_read_val(mmio: Mmio<'_, u32>) { + /// // let mmio: Mmio<'_, u32>; + /// let val: u32 = mmio.read_val(); + /// # } + /// ``` + #[inline] + fn read_val(self) -> Self::Target + where + Self::Backend: IoCapable, + Self::Target: Sized, + { + Self::Backend::io_read(self.as_view()) + } + + /// Write a value to I/O. + /// + /// This only works for primitives supported by the I/O backend. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_write_val(mmio: Mmio<'_, u32>) { + /// // let mmio: Mmio<'_, u32>; + /// mmio.write_val(1u32); + /// # } + /// ``` + #[inline] + fn write_val(self, value: Self::Target) + where + Self::Backend: IoCapable, + Self::Target: Sized, + { + Self::Backend::io_write(self.as_view(), value) + } + + /// Copy-read from I/O memory. + /// + /// This is equivalent to reading from the I/O memory with byte-wise copy, although the actual + /// implementation might be more efficient. There is no atomicity guarantee. Note that for some + /// backends (e.g. `Mmio`), this can read different value compared to [`read_val`] as + /// byte-swapping is not performed. + /// + /// [`read_val`]: Io::read_val + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_read(mmio: Mmio<'_, [u8; 6]>) { + /// // let mmio: Mmio<'_, [u8; 6]>; + /// let val: [u8; 6] = mmio.copy_read(); + /// # } + /// ``` + #[inline] + fn copy_read(self) -> Self::Target + where + Self::Backend: IoCopyable, + Self::Target: Sized + FromBytes, + { + Self::Backend::copy_read(self.as_view()) + } + + /// Copy-write to I/O memory. + /// + /// This is equivalent to writing to the I/O memory with byte-wise copy, although the actual + /// implementation might be more efficient. There is no atomicity guarantee. Note that for some + /// backends (e.g. `Mmio`), this can write different value compared to [`write_val`] as + /// byte-swapping is not performed. + /// + /// [`write_val`]: Io::write_val + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_write(mmio: Mmio<'_, [u8; 6]>) { + /// // let mmio: Mmio<'_, [u8; 6]>; + /// mmio.copy_write([0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]); + /// # } + /// ``` + #[inline] + fn copy_write(self, value: Self::Target) + where + Self::Backend: IoCopyable, + Self::Target: Sized + IntoBytes, + { + Self::Backend::copy_write(self.as_view(), value); + } + + /// Copy bytes from `data` to I/O memory. + /// + /// # Panics + /// + /// This function will panic if the length of `self` differs from the length of `data`, similar + /// to [`[u8]::copy_from_slice`]. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_write(mmio: Mmio<'_, [u8]>) { + /// // let mmio: Mmio<'_, [u8]>; + /// mmio.copy_from_slice(&[0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]); + /// # } + /// ``` + #[inline] + fn copy_from_slice(self, data: &[u8]) + where + Self::Backend: IoCopyable, + Self: Io<'a, Target = [u8]>, + { + assert_eq!(self.len(), data.len()); + + // SAFETY: `data.as_ptr()` is valid for read for `self.size()` bytes. + unsafe { + Self::Backend::copy_to_io(self.as_view(), data.as_ptr()); + } + } + + /// Copy bytes from I/O memory to `data`. + /// + /// # Panics + /// + /// This function will panic if the length of `self` differs from the length of `data`, similar + /// to [`[u8]::copy_from_slice`]. + /// + /// # Examples + /// + /// ```no_run + /// # use kernel::io::*; + /// # fn test_copy_write(mmio: Mmio<'_, [u8]>) { + /// // let mmio: Mmio<'_, [u8]>; + /// let mut buf = [0; 6]; + /// mmio.copy_to_slice(&mut buf); + /// # } + /// ``` + #[inline] + fn copy_to_slice(self, data: &mut [u8]) + where + Self::Backend: IoCopyable, + Self: Io<'a, Target = [u8]>, + { + assert_eq!(self.len(), data.len()); + + // SAFETY: `data.as_mut_ptr()` is valid for write for `self.size()` bytes. + unsafe { + Self::Backend::copy_from_io(self.as_view(), data.as_mut_ptr()); + } } /// Fallible 8-bit read with runtime bounds check. #[inline(always)] - fn try_read8(&self, offset: usize) -> Result + fn try_read8(self, offset: usize) -> Result where - Self: IoCapable, + usize: IoLoc, + Self::Backend: IoCapable, { self.try_read(offset) } /// Fallible 16-bit read with runtime bounds check. #[inline(always)] - fn try_read16(&self, offset: usize) -> Result + fn try_read16(self, offset: usize) -> Result where - Self: IoCapable, + usize: IoLoc, + Self::Backend: IoCapable, { self.try_read(offset) } /// Fallible 32-bit read with runtime bounds check. #[inline(always)] - fn try_read32(&self, offset: usize) -> Result + fn try_read32(self, offset: usize) -> Result where - Self: IoCapable, + usize: IoLoc, + Self::Backend: IoCapable, { self.try_read(offset) } /// Fallible 64-bit read with runtime bounds check. #[inline(always)] - fn try_read64(&self, offset: usize) -> Result + fn try_read64(self, offset: usize) -> Result where - Self: IoCapable, + usize: IoLoc, + Self::Backend: IoCapable, { self.try_read(offset) } /// Fallible 8-bit write with runtime bounds check. #[inline(always)] - fn try_write8(&self, value: u8, offset: usize) -> Result + fn try_write8(self, value: u8, offset: usize) -> Result where - Self: IoCapable, + usize: IoLoc, + Self::Backend: IoCapable, { self.try_write(offset, value) } /// Fallible 16-bit write with runtime bounds check. #[inline(always)] - fn try_write16(&self, value: u16, offset: usize) -> Result + fn try_write16(self, value: u16, offset: usize) -> Result where - Self: IoCapable, + usize: IoLoc, + Self::Backend: IoCapable, { self.try_write(offset, value) } /// Fallible 32-bit write with runtime bounds check. #[inline(always)] - fn try_write32(&self, value: u32, offset: usize) -> Result + fn try_write32(self, value: u32, offset: usize) -> Result where - Self: IoCapable, + usize: IoLoc, + Self::Backend: IoCapable, { self.try_write(offset, value) } /// Fallible 64-bit write with runtime bounds check. #[inline(always)] - fn try_write64(&self, value: u64, offset: usize) -> Result + fn try_write64(self, value: u64, offset: usize) -> Result where - Self: IoCapable, + usize: IoLoc, + Self::Backend: IoCapable, { self.try_write(offset, value) } /// Infallible 8-bit read with compile-time bounds check. + /// + /// `offset` should be constant. #[inline(always)] - fn read8(&self, offset: usize) -> u8 + fn read8(self, offset: usize) -> u8 where - Self: IoKnownSize + IoCapable, + usize: IoLoc, + Self::Backend: IoCapable, { self.read(offset) } /// Infallible 16-bit read with compile-time bounds check. + /// + /// `offset` should be constant. #[inline(always)] - fn read16(&self, offset: usize) -> u16 + fn read16(self, offset: usize) -> u16 where - Self: IoKnownSize + IoCapable, + usize: IoLoc, + Self::Backend: IoCapable, { self.read(offset) } /// Infallible 32-bit read with compile-time bounds check. + /// + /// `offset` should be constant. #[inline(always)] - fn read32(&self, offset: usize) -> u32 + fn read32(self, offset: usize) -> u32 where - Self: IoKnownSize + IoCapable, + usize: IoLoc, + Self::Backend: IoCapable, { self.read(offset) } /// Infallible 64-bit read with compile-time bounds check. + /// + /// `offset` should be constant. #[inline(always)] - fn read64(&self, offset: usize) -> u64 + fn read64(self, offset: usize) -> u64 where - Self: IoKnownSize + IoCapable, + usize: IoLoc, + Self::Backend: IoCapable, { self.read(offset) } /// Infallible 8-bit write with compile-time bounds check. + /// + /// `offset` should be constant. #[inline(always)] - fn write8(&self, value: u8, offset: usize) + fn write8(self, value: u8, offset: usize) where - Self: IoKnownSize + IoCapable, + usize: IoLoc, + Self::Backend: IoCapable, { self.write(offset, value) } /// Infallible 16-bit write with compile-time bounds check. + /// + /// `offset` should be constant. #[inline(always)] - fn write16(&self, value: u16, offset: usize) + fn write16(self, value: u16, offset: usize) where - Self: IoKnownSize + IoCapable, + usize: IoLoc, + Self::Backend: IoCapable, { self.write(offset, value) } /// Infallible 32-bit write with compile-time bounds check. + /// + /// `offset` should be constant. #[inline(always)] - fn write32(&self, value: u32, offset: usize) + fn write32(self, value: u32, offset: usize) where - Self: IoKnownSize + IoCapable, + usize: IoLoc, + Self::Backend: IoCapable, { self.write(offset, value) } /// Infallible 64-bit write with compile-time bounds check. + /// + /// `offset` should be constant. #[inline(always)] - fn write64(&self, value: u64, offset: usize) + fn write64(self, value: u64, offset: usize) where - Self: IoKnownSize + IoCapable, + usize: IoLoc, + Self::Backend: IoCapable, { self.write(offset, value) } @@ -416,9 +827,10 @@ fn write64(&self, value: u64, offset: usize) /// use kernel::io::{ /// Io, /// Mmio, + /// Region, /// }; /// - /// fn do_reads(io: &Mmio) -> Result { + /// fn do_reads(io: Mmio<'_, Region>) -> Result { /// // 32-bit read from address `0x10`. /// let v: u32 = io.try_read(0x10)?; /// @@ -429,15 +841,13 @@ fn write64(&self, value: u64, offset: usize) /// } /// ``` #[inline(always)] - fn try_read(&self, location: L) -> Result + fn try_read(self, location: L) -> Result where - L: IoLoc, - Self: IoCapable, + L: IoLoc, + Self::Backend: IoCapable, { - let address = self.io_addr::(location.offset())?; - - // SAFETY: `address` has been validated by `io_addr`. - Ok(unsafe { self.io_read(address) }.into()) + let view = io_view::(self, location.offset())?; + Ok(Self::Backend::io_read(view).into()) } /// Generic fallible write with runtime bounds check. @@ -450,9 +860,10 @@ fn try_read(&self, location: L) -> Result /// use kernel::io::{ /// Io, /// Mmio, + /// Region, /// }; /// - /// fn do_writes(io: &Mmio) -> Result { + /// fn do_writes(io: Mmio<'_, Region>) -> Result { /// // 32-bit write of value `1` at address `0x10`. /// io.try_write(0x10, 1u32)?; /// @@ -463,17 +874,14 @@ fn try_read(&self, location: L) -> Result /// } /// ``` #[inline(always)] - fn try_write(&self, location: L, value: T) -> Result + fn try_write(self, location: L, value: T) -> Result where - L: IoLoc, - Self: IoCapable, + L: IoLoc, + Self::Backend: IoCapable, { - let address = self.io_addr::(location.offset())?; + let view = io_view::(self, location.offset())?; let io_value = value.into(); - - // SAFETY: `address` has been validated by `io_addr`. - unsafe { self.io_write(io_value, address) } - + Self::Backend::io_write(view, io_value); Ok(()) } @@ -488,6 +896,7 @@ fn try_write(&self, location: L, value: T) -> Result /// register, /// Io, /// Mmio, + /// Region, /// }; /// /// register! { @@ -503,17 +912,17 @@ fn try_write(&self, location: L, value: T) -> Result /// } /// } /// - /// fn do_write_reg(io: &Mmio) -> Result { + /// fn do_write_reg(io: Mmio<'_, Region>) -> Result { /// /// io.try_write_reg(VERSION::new(1, 0)) /// } /// ``` #[inline(always)] - fn try_write_reg(&self, value: V) -> Result + fn try_write_reg(self, value: V) -> Result where - L: IoLoc, - V: LocatedRegister, - Self: IoCapable, + L: IoLoc, + V: LocatedRegister, + Self::Backend: IoCapable, { let (location, value) = value.into_io_op(); @@ -533,29 +942,27 @@ fn try_write_reg(&self, value: V) -> Result /// use kernel::io::{ /// Io, /// Mmio, + /// Region, /// }; /// - /// fn do_update(io: &Mmio<0x1000>) -> Result { + /// fn do_update(io: Mmio<'_, Region<0x1000>>) -> Result { /// io.try_update(0x10, |v: u32| { /// v + 1 /// }) /// } /// ``` #[inline(always)] - fn try_update(&self, location: L, f: F) -> Result + fn try_update(self, location: L, f: F) -> Result where - L: IoLoc, - Self: IoCapable, + L: IoLoc, + Self::Backend: IoCapable, F: FnOnce(T) -> T, { - let address = self.io_addr::(location.offset())?; + let view = io_view::(self, location.offset())?; - // SAFETY: `address` has been validated by `io_addr`. - let value: T = unsafe { self.io_read(address) }.into(); + let value: T = Self::Backend::io_read(view).into(); let io_value = f(value).into(); - - // SAFETY: `address` has been validated by `io_addr`. - unsafe { self.io_write(io_value, address) } + Self::Backend::io_write(view, io_value); Ok(()) } @@ -570,9 +977,10 @@ fn try_update(&self, location: L, f: F) -> Result /// use kernel::io::{ /// Io, /// Mmio, + /// Region, /// }; /// - /// fn do_reads(io: &Mmio<0x1000>) { + /// fn do_reads(io: Mmio<'_, Region<0x1000>>) { /// // 32-bit read from address `0x10`. /// let v: u32 = io.read(0x10); /// @@ -581,15 +989,13 @@ fn try_update(&self, location: L, f: F) -> Result /// } /// ``` #[inline(always)] - fn read(&self, location: L) -> T + fn read(self, location: L) -> T where - L: IoLoc, - Self: IoKnownSize + IoCapable, + L: IoLoc, + Self::Backend: IoCapable, { - let address = self.io_addr_assert::(location.offset()); - - // SAFETY: `address` has been validated by `io_addr_assert`. - unsafe { self.io_read(address) }.into() + let view = io_view_assert::(self, location.offset()); + Self::Backend::io_read(view).into() } /// Generic infallible write with compile-time bounds check. @@ -602,9 +1008,10 @@ fn read(&self, location: L) -> T /// use kernel::io::{ /// Io, /// Mmio, + /// Region, /// }; /// - /// fn do_writes(io: &Mmio<0x1000>) { + /// fn do_writes(io: Mmio<'_, Region<0x1000>>) { /// // 32-bit write of value `1` at address `0x10`. /// io.write(0x10, 1u32); /// @@ -613,16 +1020,14 @@ fn read(&self, location: L) -> T /// } /// ``` #[inline(always)] - fn write(&self, location: L, value: T) + fn write(self, location: L, value: T) where - L: IoLoc, - Self: IoKnownSize + IoCapable, + L: IoLoc, + Self::Backend: IoCapable, { - let address = self.io_addr_assert::(location.offset()); + let view = io_view_assert::(self, location.offset()); let io_value = value.into(); - - // SAFETY: `address` has been validated by `io_addr_assert`. - unsafe { self.io_write(io_value, address) } + Self::Backend::io_write(view, io_value); } /// Generic infallible write of a fully-located register value. @@ -636,6 +1041,7 @@ fn write(&self, location: L, value: T) /// register, /// Io, /// Mmio, + /// Region, /// }; /// /// register! { @@ -651,16 +1057,16 @@ fn write(&self, location: L, value: T) /// } /// } /// - /// fn do_write_reg(io: &Mmio<0x1000>) { + /// fn do_write_reg(io: Mmio<'_, Region<0x1000>>) { /// io.write_reg(VERSION::new(1, 0)); /// } /// ``` #[inline(always)] - fn write_reg(&self, value: V) + fn write_reg(self, value: V) where - L: IoLoc, - V: LocatedRegister, - Self: IoKnownSize + IoCapable, + L: IoLoc, + V: LocatedRegister, + Self::Backend: IoCapable, { let (location, value) = value.into_io_op(); @@ -680,143 +1086,208 @@ fn write_reg(&self, value: V) /// use kernel::io::{ /// Io, /// Mmio, + /// Region, /// }; /// - /// fn do_update(io: &Mmio<0x1000>) { + /// fn do_update(io: Mmio<'_, Region<0x1000>>) { /// io.update(0x10, |v: u32| { /// v + 1 /// }) /// } /// ``` #[inline(always)] - fn update(&self, location: L, f: F) + fn update(self, location: L, f: F) where - L: IoLoc, - Self: IoKnownSize + IoCapable + Sized, + L: IoLoc, + Self::Backend: IoCapable, F: FnOnce(T) -> T, { - let address = self.io_addr_assert::(location.offset()); - - // SAFETY: `address` has been validated by `io_addr_assert`. - let value: T = unsafe { self.io_read(address) }.into(); + let view = io_view_assert::(self, location.offset()); + let value: T = Self::Backend::io_read(view).into(); let io_value = f(value).into(); - - // SAFETY: `address` has been validated by `io_addr_assert`. - unsafe { self.io_write(io_value, address) } + Self::Backend::io_write(view, io_value); } } -/// Trait for types with a known size at compile time. -/// -/// This trait is implemented by I/O backends that have a compile-time known size, -/// enabling the use of infallible I/O accessors with compile-time bounds checking. -/// -/// Types implementing this trait can use the infallible methods in [`Io`] trait -/// (e.g., `read8`, `write32`), which require `Self: IoKnownSize` bound. -pub trait IoKnownSize: Io { - /// Minimum usable size of this region. - const MIN_SIZE: usize; +// Blanket implementation ensures that provided methods cannot be arbitrarily overridden by +// implementers, which is relied upon for correctness and soundness. +impl<'a, T: IoBase<'a>> Io<'a> for T {} - /// Returns the absolute I/O address for a given `offset`, - /// performing compile-time bound checks. - // Always inline to optimize out error path of `build_assert`. - #[inline(always)] - fn io_addr_assert(&self, offset: usize) -> usize { - build_assert!(offset_valid::(offset, Self::MIN_SIZE)); +/// A view of memory-mapped I/O region. +/// +/// # Invariant +/// +/// `ptr` points to a valid and aligned memory-mapped I/O region for the duration lifetime `'a`. +pub struct Mmio<'a, T: ?Sized> { + ptr: *mut T, + phantom: PhantomData<&'a ()>, +} - self.addr() + offset +impl Copy for Mmio<'_, T> {} +impl Clone for Mmio<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self } } -/// Implements [`IoCapable`] on `$mmio` for `$ty` using `$read_fn` and `$write_fn`. +impl<'a, T: ?Sized> Mmio<'a, T> { + /// Create a `Mmio`, providing the accessors to the MMIO mapping. + /// + /// # Safety + /// + /// `raw` represents a valid and aligned memory-mapped I/O region while `'a` is alive. + #[inline] + pub unsafe fn from_raw(raw: MmioRaw) -> Self { + // INVARIANT: Per safety requirement. + Self { + ptr: raw.ptr, + phantom: PhantomData, + } + } +} + +// SAFETY: `Mmio<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl Send for Mmio<'_, T> {} + +// SAFETY: `Mmio<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl Sync for Mmio<'_, T> {} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for Mmio<'a, T> { + type Backend = MmioBackend; + type Target = T; + + #[inline] + fn as_view(self) -> Mmio<'a, T> { + self + } +} + +/// I/O Backend for memory-mapped I/O. +pub struct MmioBackend; + +impl IoBackend for MmioBackend { + type View<'a, T: ?Sized + KnownSize> = Mmio<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + view.ptr + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + _view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // INVARIANT: Per safety requirement, `ptr` is projection from `view`, so it is also a valid + // memory-mapped I/O region. + Mmio { + ptr, + phantom: PhantomData, + } + } +} + +/// Implements [`IoCapable`] on `$backend` for `$ty` using `$read_fn` and `$write_fn`. macro_rules! impl_mmio_io_capable { - ($mmio:ident, $(#[$attr:meta])* $ty:ty, $read_fn:ident, $write_fn:ident) => { - $(#[$attr])* - impl IoCapable<$ty> for $mmio { - unsafe fn io_read(&self, address: usize) -> $ty { - // SAFETY: By the trait invariant `address` is a valid address for MMIO operations. - unsafe { bindings::$read_fn(address as *const c_void) } + ($backend: ident, $ty:ty, $read_fn:ident, $write_fn:ident) => { + impl IoCapable<$ty> for $backend { + #[inline] + fn io_read(view: <$backend as IoBackend>::View<'_, $ty>) -> $ty { + // SAFETY: `$backend::as_ptr(view)` is a valid pointer for MMIO operations for both + // `MmioBackend` and `RelaxedMmioBackend`. + unsafe { bindings::$read_fn($backend::as_ptr(view).cast_const().cast()) } } - unsafe fn io_write(&self, value: $ty, address: usize) { - // SAFETY: By the trait invariant `address` is a valid address for MMIO operations. - unsafe { bindings::$write_fn(value, address as *mut c_void) } + #[inline] + fn io_write(view: <$backend as IoBackend>::View<'_, $ty>, value: $ty) { + // SAFETY: `$backend::as_ptr(view)` is a valid pointer for MMIO operations for both + // `MmioBackend` and `RelaxedMmioBackend`. + unsafe { bindings::$write_fn(value, $backend::as_ptr(view).cast()) } } } }; } // MMIO regions support 8, 16, and 32-bit accesses. -impl_mmio_io_capable!(Mmio, u8, readb, writeb); -impl_mmio_io_capable!(Mmio, u16, readw, writew); -impl_mmio_io_capable!(Mmio, u32, readl, writel); +impl_mmio_io_capable!(MmioBackend, u8, readb, writeb); +impl_mmio_io_capable!(MmioBackend, u16, readw, writew); +impl_mmio_io_capable!(MmioBackend, u32, readl, writel); // MMIO regions on 64-bit systems also support 64-bit accesses. -impl_mmio_io_capable!( - Mmio, - #[cfg(CONFIG_64BIT)] - u64, - readq, - writeq -); +#[cfg(CONFIG_64BIT)] +impl_mmio_io_capable!(MmioBackend, u64, readq, writeq); -impl Io for Mmio { - /// Returns the base address of this mapping. +impl IoCopyable for MmioBackend { #[inline] - fn addr(&self) -> usize { - self.0.addr() + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + // SAFETY: + // - `view.ptr` is valid MMIO memory for `view.size()` bytes. + // - `buffer` is valid for write for `view.size()` bytes. + unsafe { + bindings::memcpy_fromio(buffer.cast(), view.ptr.cast(), view.size()); + } } - /// Returns the maximum size of this mapping. #[inline] - fn maxsize(&self) -> usize { - self.0.maxsize() + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + // SAFETY: + // - `view.ptr` is valid MMIO memory for `view.size()` bytes. + // - `buffer` is valid for read for `view.size()` bytes. + unsafe { + bindings::memcpy_toio(view.ptr.cast(), buffer.cast(), view.size()); + } } } -impl IoKnownSize for Mmio { - const MIN_SIZE: usize = SIZE; -} - -impl Mmio { - /// Converts an `MmioRaw` into an `Mmio` instance, providing the accessors to the MMIO mapping. - /// - /// # Safety - /// - /// Callers must ensure that `addr` is the start of a valid I/O mapped memory region of size - /// `maxsize`. - pub unsafe fn from_raw(raw: &MmioRaw) -> &Self { - // SAFETY: `Mmio` is a transparent wrapper around `MmioRaw`. - unsafe { &*core::ptr::from_ref(raw).cast() } - } -} - -/// [`Mmio`] wrapper using relaxed accessors. +/// [`Mmio`] but using relaxed accessors. /// /// This type provides an implementation of [`Io`] that uses relaxed I/O MMIO operands instead of /// the regular ones. /// /// See [`Mmio::relaxed`] for a usage example. -#[repr(transparent)] -pub struct RelaxedMmio(Mmio); +pub struct RelaxedMmio<'a, T: ?Sized>(Mmio<'a, T>); -impl Io for RelaxedMmio { +impl Copy for RelaxedMmio<'_, T> {} +impl Clone for RelaxedMmio<'_, T> { #[inline] - fn addr(&self) -> usize { - self.0.addr() - } - - #[inline] - fn maxsize(&self) -> usize { - self.0.maxsize() + fn clone(&self) -> Self { + *self } } -impl IoKnownSize for RelaxedMmio { - const MIN_SIZE: usize = SIZE; +/// I/O Backend for memory-mapped I/O, with relaxed access semantics. +pub struct RelaxedMmioBackend; + +impl IoBackend for RelaxedMmioBackend { + type View<'a, T: ?Sized + KnownSize> = RelaxedMmio<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + MmioBackend::as_ptr(view.0) + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // SAFETY: Per safety requirement. + RelaxedMmio(unsafe { MmioBackend::project_view(view.0, ptr) }) + } } -impl Mmio { - /// Returns a [`RelaxedMmio`] reference that performs relaxed I/O operations. +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for RelaxedMmio<'a, T> { + type Backend = RelaxedMmioBackend; + type Target = T; + + #[inline] + fn as_view(self) -> RelaxedMmio<'a, T> { + self + } +} + +impl<'a, T: ?Sized> Mmio<'a, T> { + /// Returns a [`RelaxedMmio`] that performs relaxed I/O operations. /// /// Relaxed accessors do not provide ordering guarantees with respect to DMA or memory accesses /// and can be used when such ordering is not required. @@ -827,31 +1298,457 @@ impl Mmio { /// use kernel::io::{ /// Io, /// Mmio, + /// Region, /// RelaxedMmio, /// }; /// - /// fn do_io(io: &Mmio<0x100>) { + /// fn do_io(io: Mmio<'_, Region<0x100>>) { /// // The access is performed using `readl_relaxed` instead of `readl`. /// let v = io.relaxed().read32(0x10); /// } /// /// ``` - pub fn relaxed(&self) -> &RelaxedMmio { - // SAFETY: `RelaxedMmio` is `#[repr(transparent)]` over `Mmio`, so `Mmio` and - // `RelaxedMmio` have identical layout. - unsafe { core::mem::transmute(self) } + #[inline] + pub fn relaxed(self) -> RelaxedMmio<'a, T> { + RelaxedMmio(self) } } // MMIO regions support 8, 16, and 32-bit accesses. -impl_mmio_io_capable!(RelaxedMmio, u8, readb_relaxed, writeb_relaxed); -impl_mmio_io_capable!(RelaxedMmio, u16, readw_relaxed, writew_relaxed); -impl_mmio_io_capable!(RelaxedMmio, u32, readl_relaxed, writel_relaxed); +impl_mmio_io_capable!(RelaxedMmioBackend, u8, readb_relaxed, writeb_relaxed); +impl_mmio_io_capable!(RelaxedMmioBackend, u16, readw_relaxed, writew_relaxed); +impl_mmio_io_capable!(RelaxedMmioBackend, u32, readl_relaxed, writel_relaxed); // MMIO regions on 64-bit systems also support 64-bit accesses. -impl_mmio_io_capable!( - RelaxedMmio, - #[cfg(CONFIG_64BIT)] - u64, - readq_relaxed, - writeq_relaxed -); +#[cfg(CONFIG_64BIT)] +impl_mmio_io_capable!(RelaxedMmioBackend, u64, readq_relaxed, writeq_relaxed); + +/// I/O Backend for system memory. +pub struct SysMemBackend; + +impl IoBackend for SysMemBackend { + type View<'a, T: ?Sized + KnownSize> = SysMem<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + view.ptr + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + _view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // INVARIANT: Per safety requirement, `ptr` is projection from `view`, so it is also a valid + // kernel accessible memory region. + SysMem { + ptr, + phantom: PhantomData, + } + } +} + +/// Implements [`IoCapable`] on `SysMemBackend` for `$ty` using `read_volatile` and +/// `write_volatile`. +macro_rules! impl_sysmem_io_capable { + ($ty:ty) => { + impl IoCapable<$ty> for SysMemBackend { + #[inline] + fn io_read(view: SysMem<'_, $ty>) -> $ty { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using read_volatile() here so that race with hardware is well-defined. + // - Using read_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + // - The macro is only used on primitives so all bit patterns are valid. + unsafe { view.ptr.read_volatile() } + } + + #[inline] + fn io_write(view: SysMem<'_, $ty>, value: $ty) { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using write_volatile() here so that race with hardware is well-defined. + // - Using write_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + unsafe { view.ptr.write_volatile(value) } + } + } + }; +} + +impl_sysmem_io_capable!(u8); +impl_sysmem_io_capable!(u16); +impl_sysmem_io_capable!(u32); +#[cfg(CONFIG_64BIT)] +impl_sysmem_io_capable!(u64); + +impl IoCopyable for SysMemBackend { + #[inline] + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + // Use `bindings::memcpy` instead of `copy_nonoverlapping` for volatile. + // SAFETY: + // - `view.ptr` is in CPU address space and valid for read. + // - `buffer` is valid for write for `view.size()` bytes which is equal to `view.ptr.len()`. + unsafe { bindings::memcpy(buffer.cast(), view.ptr.cast(), view.ptr.len()) }; + } + + #[inline] + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + // Use `bindings::memcpy` instead of `copy_nonoverlapping` for volatile. + // SAFETY: + // - `view.ptr` is in CPU address space and valid for write. + // - `buffer` is valid for read for `view.size()` bytes which is equal to `view.ptr.len()`. + unsafe { bindings::memcpy(view.ptr.cast(), buffer.cast(), view.ptr.len()) }; + } + + #[inline] + fn copy_read(view: Self::View<'_, T>) -> T { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using read_volatile() here so that race with hardware is well-defined. + // - Using read_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + // - `T: FromBytes` so all bit patterns are valid. + unsafe { view.ptr.read_volatile() } + } + + #[inline] + fn copy_write(view: Self::View<'_, T>, value: T) { + // SAFETY: + // - Per type invariant, `ptr` is valid and aligned. + // - Using write_volatile() here so that race with hardware is well-defined. + // - Using write_volatile() here is not sound if it races with other CPU per Rust + // rules, but this is allowed per LKMM. + unsafe { view.ptr.write_volatile(value) } + } +} + +/// A view of a system memory region. +/// +/// Provides `Io` trait implementation for kernel virtual address ranges, +/// using volatile read/write to safely access shared memory that may be +/// concurrently accessed by external hardware. +/// +/// # Invariants +/// +/// `self.ptr.addr() .. self.ptr.addr() + KnownSize::size(self.ptr)` is valid and aligned kernel +/// accessible memory region for the lifetime `'a`. +pub struct SysMem<'a, T: ?Sized> { + ptr: *mut T, + phantom: PhantomData<&'a ()>, +} + +impl Copy for SysMem<'_, T> {} +impl Clone for SysMem<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +// SAFETY: `SysMem<'_, T>` is conceptually `&T`. +unsafe impl Send for SysMem<'_, T> {} + +// SAFETY: `SysMem<'_, T>` is conceptually `&T`. +unsafe impl Sync for SysMem<'_, T> {} + +impl<'a, T: ?Sized> SysMem<'a, T> { + /// Create a `SysMem` from a raw pointer. + /// + /// # Safety + /// + /// `ptr.addr() .. ptr.addr() + KnownSize::size(ptr)` must be valid and aligned kernel + /// accessible memory region for the lifetime `'a`. + #[inline] + pub unsafe fn new(ptr: *mut T) -> Self { + // INVARIANT: Per safety requirement. + Self { + ptr, + phantom: PhantomData, + } + } + + /// Obtain the raw pointer to the memory. + #[inline] + pub fn as_ptr(self) -> *mut T { + self.ptr + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for SysMem<'a, T> { + type Backend = SysMemBackend; + type Target = T; + + #[inline] + fn as_view(self) -> ::View<'a, Self::Target> { + self + } +} + +/// I/O Backend for [`IoSysMap`]. +pub struct IoSysMapBackend; + +/// Either [`Mmio`] or [`SysMem`]. +/// +/// This can be used when a piece of logic may wish to handle both MMIO or system memory but does +/// not want or cannot be generic over I/O backends. This serves a similar purpose to +/// [`include/linux/iosys-map.h`] in C. +/// +/// This type can be used like any other types that implements [`Io`]; this also include +/// [`io_project!`], [`io_read!`], [`io_write!`]. +/// +/// [`include/linux/iosys-map.h`]: srctree/include/linux/iosys-map.h +pub enum IoSysMap<'a, T: ?Sized> { + /// The view is I/O memory. + Io(Mmio<'a, T>), + /// The view is system memory. + Sys(SysMem<'a, T>), +} + +impl Copy for IoSysMap<'_, T> {} +impl Clone for IoSysMap<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl<'a, T: ?Sized> From> for IoSysMap<'a, T> { + #[inline] + fn from(value: Mmio<'a, T>) -> Self { + IoSysMap::Io(value) + } +} + +impl<'a, T: ?Sized> From> for IoSysMap<'a, T> { + #[inline] + fn from(value: SysMem<'a, T>) -> Self { + IoSysMap::Sys(value) + } +} + +impl IoBackend for IoSysMapBackend { + type View<'a, T: ?Sized + KnownSize> = IoSysMap<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: Self::View<'a, T>) -> *mut T { + match view { + IoSysMap::Io(l) => MmioBackend::as_ptr(l), + IoSysMap::Sys(r) => SysMemBackend::as_ptr(r), + } + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + match view { + // SAFETY: Per safety requirement. + IoSysMap::Io(l) => IoSysMap::Io(unsafe { MmioBackend::project_view(l, ptr) }), + // SAFETY: Per safety requirement. + IoSysMap::Sys(r) => IoSysMap::Sys(unsafe { SysMemBackend::project_view(r, ptr) }), + } + } +} + +impl IoCapable for IoSysMapBackend +where + MmioBackend: IoCapable, + SysMemBackend: IoCapable, +{ + #[inline] + fn io_read(view: Self::View<'_, T>) -> T { + match view { + IoSysMap::Io(l) => MmioBackend::io_read(l), + IoSysMap::Sys(r) => SysMemBackend::io_read(r), + } + } + + #[inline] + fn io_write<'a>(view: Self::View<'a, T>, value: T) { + match view { + IoSysMap::Io(l) => MmioBackend::io_write(l, value), + IoSysMap::Sys(r) => SysMemBackend::io_write(r, value), + } + } +} + +impl IoCopyable for IoSysMapBackend { + #[inline] + unsafe fn copy_from_io(view: Self::View<'_, [u8]>, buffer: *mut u8) { + match view { + // SAFETY: Per safety requirement. + IoSysMap::Io(l) => unsafe { MmioBackend::copy_from_io(l, buffer) }, + // SAFETY: Per safety requirement. + IoSysMap::Sys(r) => unsafe { SysMemBackend::copy_from_io(r, buffer) }, + } + } + + #[inline] + unsafe fn copy_to_io(view: Self::View<'_, [u8]>, buffer: *const u8) { + match view { + // SAFETY: Per safety requirement. + IoSysMap::Io(l) => unsafe { MmioBackend::copy_to_io(l, buffer) }, + // SAFETY: Per safety requirement. + IoSysMap::Sys(r) => unsafe { SysMemBackend::copy_to_io(r, buffer) }, + } + } + + #[inline] + fn copy_read(view: Self::View<'_, T>) -> T { + match view { + IoSysMap::Io(l) => MmioBackend::copy_read(l), + IoSysMap::Sys(r) => SysMemBackend::copy_read(r), + } + } + + #[inline] + fn copy_write(view: Self::View<'_, T>, value: T) { + match view { + IoSysMap::Io(l) => MmioBackend::copy_write(l, value), + IoSysMap::Sys(r) => SysMemBackend::copy_write(r, value), + } + } +} + +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for IoSysMap<'a, T> { + type Backend = IoSysMapBackend; + type Target = T; + + #[inline] + fn as_view(self) -> IoSysMap<'a, T> { + self + } +} + +// This helper turns associated functions to methods so it can be invoked in macro. +// Used by `io_project!()` only. +#[doc(hidden)] +#[derive(Clone, Copy)] +pub struct ProjectHelper(pub T); + +impl<'a, T> ProjectHelper +where + T: Io<'a, Backend: IoBackend = T>>, +{ + // These helper methods must not have symbols present in the binary to avoid confusion. + #[inline(always)] + pub fn as_ptr(self) -> *mut T::Target { + T::Backend::as_ptr(self.0) + } + + /// # Safety + /// + /// Same as `IoBackend::project_view` + #[inline(always)] + pub unsafe fn project_view( + self, + ptr: *mut U, + ) -> ::View<'a, U> { + // SAFETY: Per safety requirement. + unsafe { T::Backend::project_view::(self.0, ptr) } + } +} + +/// Project an I/O type to a subview of it. +/// +/// The syntax is of form `io_project!(io, proj)` where `io` is an expression to a type that +/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!). +/// +/// # Examples +/// +/// ``` +/// use kernel::io::{ +/// io_project, +/// Mmio, +/// }; +/// #[repr(C)] +/// struct MyStruct { field: u32, } +/// +/// # fn test(mmio: Mmio<'_, [MyStruct]>) -> Result { +/// // let mmio: Mmio<[MyStruct]>; +/// let field: Mmio<'_, u32> = io_project!(mmio, [try: 1].field); +/// let whole: Mmio<'_, MyStruct> = io_project!(mmio, [try: 2]); +/// let nested: Mmio<'_, u32> = io_project!(whole, .field); +/// # Ok::<(), Error>(()) } +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! io_project { + ($io:expr, $($proj:tt)*) => {{ + #[allow(unused)] + use $crate::io::IoBase as _; + let view = $crate::io::ProjectHelper($io.as_view()); + let ptr = $crate::ptr::project!( + mut view.as_ptr(), $($proj)* + ); + #[allow(unused_unsafe)] + // SAFETY: `ptr` is a projection. + unsafe { view.project_view(ptr) } + }}; +} +#[doc(inline)] +pub use crate::io_project; + +/// Read from I/O memory. +/// +/// The syntax is of form `io_read!(io, proj)` where `io` is an expression to a type that +/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!). +/// +/// # Examples +/// +/// ``` +/// #[repr(C)] +/// struct MyStruct { field: u32, } +/// +/// # fn test(mmio: kernel::io::Mmio<'_, [MyStruct]>) -> Result { +/// // let mmio: Mmio<'_, [MyStruct]>; +/// let field: u32 = kernel::io::io_read!(mmio, [try: 2].field); +/// # Ok::<(), Error>(()) } +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! io_read { + ($io:expr, $($proj:tt)*) => { + $crate::io::Io::read_val($crate::io_project!($io, $($proj)*)) + }; +} +#[doc(inline)] +pub use crate::io_read; + +/// Writes to I/O memory. +/// +/// The syntax is of form `io_write!(io, proj, val)` where `io` is an expression to a type that +/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!), +/// and `val` is the value to be written to the projected location. +/// +/// # Examples +/// +/// ``` +/// #[repr(C)] +/// struct MyStruct { field: u32, } +/// +/// # fn test(mmio: kernel::io::Mmio<'_, [MyStruct]>) -> Result { +/// // let mmio: Mmio<'_, [MyStruct]>; +/// kernel::io::io_write!(mmio, [try: 2].field, 10); +/// # Ok::<(), Error>(()) } +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! io_write { + (@parse [$io:expr] [$($proj:tt)*] [, $val:expr]) => { + $crate::io::Io::write_val($crate::io_project!($io, $($proj)*), $val) + }; + (@parse [$io:expr] [$($proj:tt)*] [.$field:tt $($rest:tt)*]) => { + $crate::io_write!(@parse [$io] [$($proj)* .$field] [$($rest)*]) + }; + (@parse [$io:expr] [$($proj:tt)*] [[$flavor:ident: $index:expr] $($rest:tt)*]) => { + $crate::io_write!(@parse [$io] [$($proj)* [$flavor: $index]] [$($rest)*]) + }; + ($io:expr, $($rest:tt)*) => { + $crate::io_write!(@parse [$io] [] [$($rest)*]) + }; +} +#[doc(inline)] +pub use crate::io_write; diff --git a/rust/kernel/io/mem.rs b/rust/kernel/io/mem.rs index fc2a3e24f8d5..32a919099dcd 100644 --- a/rust/kernel/io/mem.rs +++ b/rust/kernel/io/mem.rs @@ -2,24 +2,28 @@ //! Generic memory-mapped IO. -use core::ops::Deref; - use crate::{ device::{ Bound, Device, // }, - devres::Devres, + devres::DevresLt, io::{ self, resource::{ Region, Resource, // }, + IoBase, Mmio, + MmioBackend, MmioRaw, // }, prelude::*, + types::{ + CovariantForLt, + ForLt, // + }, }; /// An IO request for a specific device and resource. @@ -172,6 +176,19 @@ pub struct ExclusiveIoMem<'a, const SIZE: usize> { _region: Region, } +impl ForLt for ExclusiveIoMem<'static, SIZE> { + type Of<'a> = ExclusiveIoMem<'a, SIZE>; +} + +// SAFETY: `ExclusiveIoMem<'a, SIZE>` is covariant over `'a`; it holds an `IoMem<'a, SIZE>`, +// which holds `&'a Device`, which is covariant. +unsafe impl CovariantForLt for ExclusiveIoMem<'static, SIZE> {} + +/// A device-managed exclusive I/O memory region. +/// +/// See [`ExclusiveIoMem::into_devres`]. +pub type DevresExclusiveIoMem = DevresLt>; + impl<'a, const SIZE: usize> ExclusiveIoMem<'a, SIZE> { /// Creates a new `ExclusiveIoMem` instance. fn ioremap(dev: &'a Device, resource: &Resource) -> Result { @@ -198,23 +215,23 @@ fn ioremap(dev: &'a Device, resource: &Resource) -> Result { /// Consume the `ExclusiveIoMem` and register it as a device-managed resource. /// - /// The returned `Devres>` can outlive the original lifetime - /// `'a`. Access to the I/O memory is revoked when the device is unbound. - pub fn into_devres(self) -> Result>> { - // SAFETY: Casting to `'static` is sound because `Devres` guarantees the - // `ExclusiveIoMem` does not actually outlive the device -- access is revoked and the - // resource is released when the device is unbound. - let iomem: ExclusiveIoMem<'static, SIZE> = unsafe { core::mem::transmute(self) }; - let dev = iomem.iomem.dev; - Devres::new(dev, iomem) + /// The returned [`DevresExclusiveIoMem`] can outlive the original borrow and be stored in + /// driver data. Access to the I/O memory is revoked automatically when the device is unbound. + pub fn into_devres(self) -> Result> { + let dev = self.iomem.dev; + // SAFETY: `ExclusiveIoMem` only holds a device reference and an I/O mapping, both of + // which remain valid for the device's full bound scope, not just for `'a`. + unsafe { DevresLt::new(dev, self) } } } -impl Deref for ExclusiveIoMem<'_, SIZE> { - type Target = Mmio; +impl<'a, const SIZE: usize> IoBase<'a> for &'a ExclusiveIoMem<'_, SIZE> { + type Backend = MmioBackend; + type Target = super::Region; - fn deref(&self) -> &Self::Target { - &self.iomem + #[inline] + fn as_view(self) -> Mmio<'a, Self::Target> { + self.iomem.as_view() } } @@ -229,9 +246,22 @@ fn deref(&self) -> &Self::Target { /// start of the I/O memory mapped region. pub struct IoMem<'a, const SIZE: usize = 0> { dev: &'a Device, - io: MmioRaw, + io: MmioRaw>, } +impl ForLt for IoMem<'static, SIZE> { + type Of<'a> = IoMem<'a, SIZE>; +} + +// SAFETY: `IoMem<'a, SIZE>` is covariant over `'a`; it holds `&'a Device`, +// which is covariant. +unsafe impl CovariantForLt for IoMem<'static, SIZE> {} + +/// A device-managed I/O memory region. +/// +/// See [`IoMem::into_devres`]. +pub type DevresIoMem = DevresLt>; + impl<'a, const SIZE: usize> IoMem<'a, SIZE> { fn ioremap(dev: &'a Device, resource: &Resource) -> Result { // Note: Some ioremap() implementations use types that depend on the CPU @@ -264,23 +294,19 @@ fn ioremap(dev: &'a Device, resource: &Resource) -> Result { return Err(ENOMEM); } - let io = MmioRaw::new(addr as usize, size)?; - + let io = MmioRaw::new_region(addr as usize, size)?; Ok(IoMem { dev, io }) } /// Consume the `IoMem` and register it as a device-managed resource. /// - /// The returned `Devres>` can outlive the original - /// lifetime `'a`. Access to the I/O memory is revoked when the device - /// is unbound. - pub fn into_devres(self) -> Result>> { - // SAFETY: Casting to `'static` is sound because `Devres` guarantees the `IoMem` does not - // actually outlive the device -- access is revoked and the resource is released when the - // device is unbound. - let iomem: IoMem<'static, SIZE> = unsafe { core::mem::transmute(self) }; - let dev = iomem.dev; - Devres::new(dev, iomem) + /// The returned [`DevresIoMem`] can outlive the original borrow and be stored in driver data. + /// Access to the I/O memory is revoked automatically when the device is unbound. + pub fn into_devres(self) -> Result> { + let dev = self.dev; + // SAFETY: `IoMem` only holds a device reference and an I/O mapping, both of which + // remain valid for the device's full bound scope, not just for `'a`. + unsafe { DevresLt::new(dev, self) } } } @@ -291,11 +317,13 @@ fn drop(&mut self) { } } -impl Deref for IoMem<'_, SIZE> { - type Target = Mmio; +impl<'a, const SIZE: usize> IoBase<'a> for &'a IoMem<'_, SIZE> { + type Backend = MmioBackend; + type Target = super::Region; - fn deref(&self) -> &Self::Target { + #[inline] + fn as_view(self) -> Mmio<'a, Self::Target> { // SAFETY: Safe as by the invariant of `IoMem`. - unsafe { Mmio::from_raw(&self.io) } + unsafe { Mmio::from_raw(self.io) } } } diff --git a/rust/kernel/io/poll.rs b/rust/kernel/io/poll.rs index 75d1b3e8596c..d75f2fcf46f2 100644 --- a/rust/kernel/io/poll.rs +++ b/rust/kernel/io/poll.rs @@ -48,13 +48,14 @@ /// use kernel::io::{ /// Io, /// Mmio, +/// Region, /// poll::read_poll_timeout, // /// }; /// use kernel::time::Delta; /// /// const HW_READY: u16 = 0x01; /// -/// fn wait_for_hardware(io: &Mmio) -> Result { +/// fn wait_for_hardware(io: Mmio<'_, Region>) -> Result { /// read_poll_timeout( /// // The `op` closure reads the value of a specific status register. /// || io.try_read16(0x1000), @@ -135,13 +136,14 @@ pub fn read_poll_timeout( /// use kernel::io::{ /// Io, /// Mmio, +/// Region, /// poll::read_poll_timeout_atomic, // /// }; /// use kernel::time::Delta; /// /// const HW_READY: u16 = 0x01; /// -/// fn wait_for_hardware(io: &Mmio) -> Result { +/// fn wait_for_hardware(io: Mmio<'_, Region>) -> Result { /// read_poll_timeout_atomic( /// // The `op` closure reads the value of a specific status register. /// || io.try_read16(0x1000), diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs index f924c7c7c1db..03dfd2ff48c7 100644 --- a/rust/kernel/io/register.rs +++ b/rust/kernel/io/register.rs @@ -58,7 +58,7 @@ //! }, //! num::Bounded, //! }; -//! # use kernel::io::Mmio; +//! # use kernel::io::{Mmio, Region}; //! # register! { //! # pub BOOT_0(u32) @ 0x00000100 { //! # 15:8 vendor_id; @@ -66,7 +66,7 @@ //! # 3:0 minor_revision; //! # } //! # } -//! # fn test(io: &Mmio<0x1000>) { +//! # fn test(io: Mmio<'_, Region<0x1000>>) { //! # fn obtain_vendor_id() -> u8 { 0xff } //! //! // Read from the register's defined offset (0x100). @@ -113,6 +113,8 @@ io::IoLoc, // }; +use super::Region; + /// Trait implemented by all registers. pub trait Register: Sized { /// Backing primitive type of the register. @@ -129,7 +131,7 @@ pub trait FixedRegister: Register {} /// Allows `()` to be used as the `location` parameter of [`Io::write`](super::Io::write) when /// passing a [`FixedRegister`] value. -impl IoLoc for () +impl IoLoc, T> for () where T: FixedRegister, { @@ -143,7 +145,7 @@ fn offset(self) -> usize { /// A [`FixedRegister`] carries its location in its type. Thus `FixedRegister` values can be used /// as an [`IoLoc`]. -impl IoLoc for T +impl IoLoc, T> for T where T: FixedRegister, { @@ -168,7 +170,7 @@ pub const fn new() -> Self { } } -impl IoLoc for FixedRegisterLoc +impl IoLoc, T> for FixedRegisterLoc where T: FixedRegister, { @@ -239,7 +241,7 @@ const fn offset(self) -> usize { } } -impl IoLoc for RelativeRegisterLoc +impl IoLoc, T> for RelativeRegisterLoc where T: RelativeRegister, B: RegisterBase + ?Sized, @@ -283,7 +285,7 @@ pub fn try_new(idx: usize) -> Option { } } -impl IoLoc for RegisterArrayLoc +impl IoLoc, T> for RegisterArrayLoc where T: RegisterArray, { @@ -370,7 +372,7 @@ pub fn try_at(self, idx: usize) -> Option> { } } -impl IoLoc for RelativeRegisterArrayLoc +impl IoLoc, T> for RelativeRegisterArrayLoc where T: RelativeRegisterArray, B: RegisterBase + ?Sized, @@ -387,18 +389,18 @@ fn offset(self) -> usize { /// which to write it. /// /// Implementors can be used with [`Io::write_reg`](super::Io::write_reg). -pub trait LocatedRegister { +pub trait LocatedRegister { /// Register value to write. type Value: Register; /// Full location information at which to write the value. - type Location: IoLoc; + type Location: IoLoc; /// Consumes `self` and returns a `(location, value)` tuple describing a valid I/O write /// operation. fn into_io_op(self) -> (Self::Location, Self::Value); } -impl LocatedRegister for T +impl LocatedRegister> for T where T: FixedRegister, { @@ -444,16 +446,16 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// Io, /// }, /// }; -/// # use kernel::io::Mmio; +/// # use kernel::io::{Mmio, Region}; /// /// register! { /// FIXED_REG(u32) @ 0x100 { -/// 16:8 high_byte; +/// 15:8 high_byte; /// 7:0 low_byte; /// } /// } /// -/// # fn test(io: &Mmio<0x1000>) { +/// # fn test(io: Mmio<'_, Region<0x1000>>) { /// let val = io.read(FIXED_REG); /// /// // Write from an already-existing value. @@ -557,7 +559,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// Io, /// }, /// }; -/// # use kernel::io::Mmio; +/// # use kernel::io::{Mmio, Region}; /// /// // Type used to identify the base. /// pub struct CpuCtlBase; @@ -582,7 +584,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// } /// } /// -/// # fn test(io: Mmio<0x1000>) { +/// # fn test(io: Mmio<'_, Region<0x1000>>) { /// // Read the status of `Cpu0`. /// let cpu0_started = io.read(CPU_CTL::of::()); /// @@ -599,7 +601,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// } /// } /// -/// # fn test2(io: Mmio<0x1000>) { +/// # fn test2(io: Mmio<'_, Region<0x1000>>) { /// // Start the aliased `CPU0`, leaving its other fields untouched. /// io.update(CPU_CTL_ALIAS::of::(), |r| r.with_alias_start(true)); /// # } @@ -636,7 +638,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// Io, /// }, /// }; -/// # use kernel::io::Mmio; +/// # use kernel::io::{Mmio, Region}; /// # fn get_scratch_idx() -> usize { /// # 0x15 /// # } @@ -649,7 +651,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// } /// } /// -/// # fn test(io: &Mmio<0x1000>) +/// # fn test(io: Mmio<'_, Region<0x1000>>) /// # -> Result<(), Error>{ /// // Read scratch register 0, i.e. I/O address `0x80`. /// let scratch_0 = io.read(SCRATCH::at(0)).value(); @@ -722,7 +724,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// Io, /// }, /// }; -/// # use kernel::io::Mmio; +/// # use kernel::io::{Mmio, Region}; /// # fn get_scratch_idx() -> usize { /// # 0x15 /// # } @@ -750,7 +752,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// } /// } /// -/// # fn test(io: &Mmio<0x1000>) -> Result<(), Error> { +/// # fn test(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> { /// // Read scratch register 0 of CPU0. /// let scratch = io.read(CPU_SCRATCH::of::().at(0)); /// @@ -792,7 +794,7 @@ fn into_io_op(self) -> (FixedRegisterLoc, T) { /// } /// } /// -/// # fn test2(io: &Mmio<0x1000>) -> Result<(), Error> { +/// # fn test2(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> { /// let cpu0_status = io.read(CPU_FIRMWARE_STATUS::of::()).status(); /// # Ok(()) /// # } @@ -802,6 +804,9 @@ macro_rules! register { // Entry point for the macro, allowing multiple registers to be defined in one call. // It matches all possible register declaration patterns to dispatch them to corresponding // `@reg` rule that defines a single register. + // + // TODO: change `alias:ident` to `alias:path` once relative registers are replaced by I/O + // projections. ( $( $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) @@ -830,12 +835,12 @@ macro_rules! register { ) => { $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); $crate::register!(@io_base $name($storage) @ $offset); - $crate::register!(@io_fixed $(#[$attr])* $vis $name($storage)); + $crate::register!(@io_fixed $(#[$attr])* $vis $name); }; // Creates an alias register of fixed offset register `alias` with its own fields. ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:ident + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:path { $($fields:tt)* } ) => { $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); @@ -843,7 +848,7 @@ macro_rules! register { @io_base $name($storage) @ <$alias as $crate::io::register::Register>::OFFSET ); - $crate::register!(@io_fixed $(#[$attr])* $vis $name($storage)); + $crate::register!(@io_fixed $(#[$attr])* $vis $name); }; // Creates a register at a relative offset from a base address provider. @@ -853,7 +858,7 @@ macro_rules! register { ) => { $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); $crate::register!(@io_base $name($storage) @ $offset); - $crate::register!(@io_relative $vis $name($storage) @ $base); + $crate::register!(@io_relative $name @ $base); }; // Creates an alias register of relative offset register `alias` with its own fields. @@ -865,7 +870,7 @@ macro_rules! register { $crate::register!( @io_base $name($storage) @ <$alias as $crate::io::register::Register>::OFFSET ); - $crate::register!(@io_relative $vis $name($storage) @ $base); + $crate::register!(@io_relative $name @ $base); }; // Creates an array of registers at a fixed offset of the MMIO space. @@ -877,7 +882,7 @@ macro_rules! register { $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); $crate::register!(@io_base $name($storage) @ $offset); - $crate::register!(@io_array $vis $name($storage) [ $size, stride = $stride ]); + $crate::register!(@io_array $name [ $size, stride = $stride ]); }; // Shortcut for contiguous array of registers (stride == size of element). @@ -886,14 +891,15 @@ macro_rules! register { { $($fields:tt)* } ) => { $crate::register!( - $(#[$attr])* $vis $name($storage) [ $size, stride = ::core::mem::size_of::<$storage>() ] + @reg $(#[$attr])* $vis $name($storage) + [ $size, stride = ::core::mem::size_of::<$storage>() ] @ $offset { $($fields)* } ); }; // Creates an alias of register `idx` of array of registers `alias` with its own fields. ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:ident [ $idx:expr ] + @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:path [ $idx:expr ] { $($fields:tt)* } ) => { $crate::build_assert::static_assert!( @@ -906,7 +912,7 @@ macro_rules! register { <$alias as $crate::io::register::Register>::OFFSET + $idx * <$alias as $crate::io::register::RegisterArray>::STRIDE ); - $crate::register!(@io_fixed $(#[$attr])* $vis $name($storage)); + $crate::register!(@io_fixed $(#[$attr])* $vis $name); }; // Creates an array of registers at a relative offset from a base address provider. @@ -919,9 +925,7 @@ macro_rules! register { $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); $crate::register!(@io_base $name($storage) @ $offset); - $crate::register!( - @io_relative_array $vis $name($storage) [ $size, stride = $stride ] @ $base + $offset - ); + $crate::register!(@io_relative_array $name [ $size, stride = $stride ] @ $base); }; // Shortcut for contiguous array of relative registers (stride == size of element). @@ -930,7 +934,8 @@ macro_rules! register { @ $base:ident + $offset:literal { $($fields:tt)* } ) => { $crate::register!( - $(#[$attr])* $vis $name($storage) [ $size, stride = ::core::mem::size_of::<$storage>() ] + @reg $(#[$attr])* $vis $name($storage) + [ $size, stride = ::core::mem::size_of::<$storage>() ] @ $base + $offset { $($fields)* } ); }; @@ -951,7 +956,7 @@ macro_rules! register { <$alias as $crate::io::register::Register>::OFFSET + $idx * <$alias as $crate::io::register::RegisterArray>::STRIDE ); - $crate::register!(@io_relative $vis $name($storage) @ $base); + $crate::register!(@io_relative $name @ $base); }; // Generates the bitfield for the register. @@ -977,7 +982,7 @@ impl $crate::io::register::Register for $name { }; // Implementations of fixed registers. - (@io_fixed $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty)) => { + (@io_fixed $(#[$attr:meta])* $vis:vis $name:ident) => { impl $crate::io::register::FixedRegister for $name {} $(#[$attr])* @@ -986,7 +991,7 @@ impl $crate::io::register::FixedRegister for $name {} }; // Implementations of relative registers. - (@io_relative $vis:vis $name:ident ($storage:ty) @ $base:ident) => { + (@io_relative $name:ident @ $base:ident) => { impl $crate::io::register::WithBase for $name { type BaseFamily = $base; } @@ -995,7 +1000,7 @@ impl $crate::io::register::RelativeRegister for $name {} }; // Implementations of register arrays. - (@io_array $vis:vis $name:ident ($storage:ty) [ $size:expr, stride = $stride:expr ]) => { + (@io_array $name:ident [ $size:expr, stride = $stride:expr ]) => { impl $crate::io::register::Array for $name {} impl $crate::io::register::RegisterArray for $name { @@ -1006,8 +1011,7 @@ impl $crate::io::register::RegisterArray for $name { // Implementations of relative array registers. ( - @io_relative_array $vis:vis $name:ident ($storage:ty) [ $size:expr, stride = $stride:expr ] - @ $base:ident + $offset:literal + @io_relative_array $name:ident [ $size:expr, stride = $stride:expr ] @ $base:ident ) => { impl $crate::io::register::WithBase for $name { type BaseFamily = $base; diff --git a/rust/kernel/irq.rs b/rust/kernel/irq.rs index 20abd4056655..09ef1e7f853c 100644 --- a/rust/kernel/irq.rs +++ b/rust/kernel/irq.rs @@ -8,7 +8,7 @@ //! The current abstractions handle IRQ requests and handlers, i.e.: it allows //! drivers to register a handler for a given IRQ line. //! -//! C header: [`include/linux/device.h`](srctree/include/linux/interrupt.h) +//! C header: [`include/linux/interrupt.h`](srctree/include/linux/interrupt.h) /// Flags to be used when registering IRQ handlers. mod flags; diff --git a/rust/kernel/irq/request.rs b/rust/kernel/irq/request.rs index f425fe12f7c8..c1c6525a676a 100644 --- a/rust/kernel/irq/request.rs +++ b/rust/kernel/irq/request.rs @@ -5,16 +5,21 @@ //! [`ThreadedRegistration`], which allow users to register handlers for a given //! IRQ line. -use core::marker::PhantomPinned; +use core::marker::{ + PhantomData, + PhantomPinned, // +}; -use crate::alloc::Allocator; -use crate::device::{Bound, Device}; -use crate::devres::Devres; -use crate::error::to_result; -use crate::irq::flags::Flags; -use crate::prelude::*; -use crate::str::CStr; -use crate::sync::Arc; +use crate::{ + device::{ + Bound, + Device, // + }, + error::to_result, + irq::flags::Flags, + prelude::*, + str::CStr, +}; /// The value that can be returned from a [`Handler`] or a [`ThreadedHandler`]. #[repr(u32)] @@ -27,7 +32,7 @@ pub enum IrqReturn { } /// Callbacks for an IRQ handler. -pub trait Handler: Sync + 'static { +pub trait Handler: Sync { /// The hard IRQ handler. /// /// This is executed in interrupt context, hence all corresponding @@ -36,73 +41,20 @@ pub trait Handler: Sync + 'static { /// All work that does not necessarily need to be executed from /// interrupt context, should be deferred to a threaded handler. /// See also [`ThreadedRegistration`]. - fn handle(&self, device: &Device) -> IrqReturn; + fn handle(&self) -> IrqReturn; } -impl Handler for Arc { - fn handle(&self, device: &Device) -> IrqReturn { - T::handle(self, device) - } -} - -impl Handler for Box { - fn handle(&self, device: &Device) -> IrqReturn { - T::handle(self, device) - } -} - -/// # Invariants -/// -/// - `self.irq` is the same as the one passed to `request_{threaded}_irq`. -/// - `cookie` was passed to `request_{threaded}_irq` as the cookie. It is guaranteed to be unique -/// by the type system, since each call to `new` will return a different instance of -/// `Registration`. -#[pin_data(PinnedDrop)] -struct RegistrationInner { - irq: u32, - cookie: *mut c_void, -} - -impl RegistrationInner { - fn synchronize(&self) { - // SAFETY: safe as per the invariants of `RegistrationInner` - unsafe { bindings::synchronize_irq(self.irq) }; - } -} - -#[pinned_drop] -impl PinnedDrop for RegistrationInner { - fn drop(self: Pin<&mut Self>) { - // SAFETY: - // - // Safe as per the invariants of `RegistrationInner` and: - // - // - The containing struct is `!Unpin` and was initialized using - // pin-init, so it occupied the same memory location for the entirety of - // its lifetime. - // - // Notice that this will block until all handlers finish executing, - // i.e.: at no point will &self be invalid while the handler is running. - unsafe { bindings::free_irq(self.irq, self.cookie) }; - } -} - -// SAFETY: We only use `inner` on drop, which called at most once with no -// concurrent access. -unsafe impl Sync for RegistrationInner {} - -// SAFETY: It is safe to send `RegistrationInner` across threads. -unsafe impl Send for RegistrationInner {} - /// A request for an IRQ line for a given device. /// /// # Invariants /// /// - `ìrq` is the number of an interrupt source of `dev`. -/// - `irq` has not been registered yet. +/// - `irq` has not been registered yet; this is consumed by [`Registration::new()`]. pub struct IrqRequest<'a> { - dev: &'a Device, irq: u32, + /// Proves the device is bound at registration time and ties `'a` to the device's bound + /// lifetime, ensuring the [`Registration`] cannot outlive it. + _dev: PhantomData<&'a Device>, } impl<'a> IrqRequest<'a> { @@ -111,12 +63,16 @@ impl<'a> IrqRequest<'a> { /// # Safety /// /// - `irq` should be a valid IRQ number for `dev`. - pub(crate) unsafe fn new(dev: &'a Device, irq: u32) -> Self { + pub(crate) unsafe fn new(_dev: &'a Device, irq: u32) -> Self { // INVARIANT: `irq` is a valid IRQ number for `dev`. - IrqRequest { dev, irq } + IrqRequest { + irq, + _dev: PhantomData, + } } /// Returns the IRQ number of an [`IrqRequest`]. + #[inline] pub fn irq(&self) -> u32 { self.irq } @@ -139,10 +95,18 @@ pub fn irq(&self) -> u32 { /// [`Completion::wait_for_completion()`]: kernel::sync::Completion::wait_for_completion /// /// ``` -/// use kernel::device::{Bound, Device}; -/// use kernel::irq::{self, Flags, IrqRequest, IrqReturn, Registration}; -/// use kernel::prelude::*; -/// use kernel::sync::{Arc, Completion}; +/// use core::pin::Pin; +/// use kernel::{ +/// irq::{ +/// self, +/// Flags, +/// IrqRequest, +/// IrqReturn, +/// Registration, +/// }, +/// prelude::*, +/// sync::Completion, +/// }; /// /// // Data shared between process and IRQ context. /// #[pin_data] @@ -153,7 +117,7 @@ pub fn irq(&self) -> u32 { /// /// impl irq::Handler for Data { /// // Executed in IRQ context. -/// fn handle(&self, _dev: &Device) -> IrqReturn { +/// fn handle(&self) -> IrqReturn { /// self.completion.complete_all(); /// IrqReturn::Handled /// } @@ -163,12 +127,21 @@ pub fn irq(&self) -> u32 { /// // /// // This runs in process context and assumes `request` was previously acquired from a device. /// fn register_irq( -/// handler: impl PinInit, /// request: IrqRequest<'_>, -/// ) -> Result>> { -/// let registration = Registration::new(request, Flags::SHARED, c"my_device", handler); +/// ) -> Result>>> { +/// // SAFETY: The returned Registration is not leaked. +/// let registration = unsafe { +/// Registration::new( +/// request, +/// Flags::SHARED, +/// c"my_device", +/// try_pin_init!(Data { +/// completion <- Completion::new(), +/// }? Error), +/// ) +/// }; /// -/// let registration = Arc::pin_init(registration, GFP_KERNEL)?; +/// let registration = KBox::pin_init(registration, GFP_KERNEL)?; /// /// registration.handler().completion.wait_for_completion(); /// @@ -179,11 +152,10 @@ pub fn irq(&self) -> u32 { /// /// # Invariants /// -/// * We own an irq handler whose cookie is a pointer to `Self`. -#[pin_data] -pub struct Registration { - #[pin] - inner: Devres, +/// * We own an irq handler registered via `request_irq` whose cookie is a pointer to `Self`. +#[pin_data(PinnedDrop)] +pub struct Registration<'a, T: Handler> { + request: IrqRequest<'a>, #[pin] handler: T, @@ -194,44 +166,46 @@ pub struct Registration { _pin: PhantomPinned, } -impl Registration { +impl<'a, T: Handler> Registration<'a, T> { /// Registers the IRQ handler with the system for the given IRQ number. - pub fn new<'a>( + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the returned [`Registration`] or otherwise prevent its + /// [`Drop`] implementation from running. + pub unsafe fn new( request: IrqRequest<'a>, flags: Flags, name: &'static CStr, handler: impl PinInit + 'a, - ) -> impl PinInit + 'a { + ) -> impl PinInit + 'a + where + T: 'a, + { + // INVARIANT: If initialization completes successfully, we own an IRQ handler registered + // via `request_irq` whose cookie is a pointer to `Self`. try_pin_init!(&this in Self { handler <- handler, - inner <- Devres::new( - request.dev, - try_pin_init!(RegistrationInner { - // INVARIANT: `this` is a valid pointer to the `Registration` instance - cookie: this.as_ptr().cast::(), - irq: { - // SAFETY: - // - The callbacks are valid for use with request_irq. - // - If this succeeds, the slot is guaranteed to be valid until the - // destructor of Self runs, which will deregister the callbacks - // before the memory location becomes invalid. - // - When request_irq is called, everything that handle_irq_callback will - // touch has already been initialized, so it's safe for the callback to - // be called immediately. - to_result(unsafe { - bindings::request_irq( - request.irq, - Some(handle_irq_callback::), - flags.into_inner(), - name.as_char_ptr(), - this.as_ptr().cast::(), - ) - })?; - request.irq - } - }) - ), + request, _pin: PhantomPinned, + _: { + // SAFETY: + // - The callbacks are valid for use with request_irq. + // - If this succeeds, the slot is guaranteed to be valid until the destructor of + // Self runs, which will deregister the callbacks before the memory location + // becomes invalid. + // - All fields are already initialized, so it's safe for the callback to be + // called immediately. + to_result(unsafe { + bindings::request_irq( + request.irq, + Some(handle_irq_callback::), + flags.into_inner(), + name.as_char_ptr(), + this.as_ptr().cast::(), + ) + })?; + }, }) } @@ -241,19 +215,25 @@ pub fn handler(&self) -> &T { } /// Wait for pending IRQ handlers on other CPUs. - /// - /// This will attempt to access the inner [`Devres`] container. - pub fn try_synchronize(&self) -> Result { - let inner = self.inner.try_access().ok_or(ENODEV)?; - inner.synchronize(); - Ok(()) + #[inline] + pub fn synchronize(&self) { + // SAFETY: `self.request.irq` is a valid registered IRQ number (type invariant). + unsafe { bindings::synchronize_irq(self.request.irq) }; } +} - /// Wait for pending IRQ handlers on other CPUs. - pub fn synchronize(&self, dev: &Device) -> Result { - let inner = self.inner.access(dev)?; - inner.synchronize(); - Ok(()) +#[pinned_drop] +impl PinnedDrop for Registration<'_, T> { + fn drop(self: Pin<&mut Self>) { + // SAFETY: The cookie was set to a pointer to `Self` in `Registration::new()`. This blocks + // until all in-flight handlers complete, so no references to `self` remain after this + // returns. + unsafe { + bindings::free_irq( + self.request.irq, + core::ptr::from_mut::(self.get_unchecked_mut()).cast::(), + ) + }; } } @@ -261,13 +241,11 @@ pub fn synchronize(&self, dev: &Device) -> Result { /// /// This function should be only used as the callback in `request_irq`. unsafe extern "C" fn handle_irq_callback(_irq: i32, ptr: *mut c_void) -> c_uint { - // SAFETY: `ptr` is a pointer to `Registration` set in `Registration::new` - let registration = unsafe { &*(ptr as *const Registration) }; - // SAFETY: The irq callback is removed before the device is unbound, so the fact that the irq - // callback is running implies that the device has not yet been unbound. - let device = unsafe { registration.inner.device().as_bound() }; + let ptr = ptr.cast_const().cast::>(); + // SAFETY: `ptr` is a pointer to `Registration<'_, T>` set in `Registration::new()`. + let registration = unsafe { &*ptr }; - T::handle(®istration.handler, device) as c_uint + T::handle(®istration.handler) as c_uint } /// The value that can be returned from [`ThreadedHandler::handle`]. @@ -284,7 +262,7 @@ pub enum ThreadedIrqReturn { } /// Callbacks for a threaded IRQ handler. -pub trait ThreadedHandler: Sync + 'static { +pub trait ThreadedHandler: Sync { /// The hard IRQ handler. /// /// This is executed in interrupt context, hence all corresponding @@ -293,8 +271,7 @@ pub trait ThreadedHandler: Sync + 'static { /// handler, i.e. [`ThreadedHandler::handle_threaded`]. /// /// The default implementation returns [`ThreadedIrqReturn::WakeThread`]. - #[expect(unused_variables)] - fn handle(&self, device: &Device) -> ThreadedIrqReturn { + fn handle(&self) -> ThreadedIrqReturn { ThreadedIrqReturn::WakeThread } @@ -302,27 +279,7 @@ fn handle(&self, device: &Device) -> ThreadedIrqReturn { /// /// This is executed in process context. The kernel creates a dedicated /// `kthread` for this purpose. - fn handle_threaded(&self, device: &Device) -> IrqReturn; -} - -impl ThreadedHandler for Arc { - fn handle(&self, device: &Device) -> ThreadedIrqReturn { - T::handle(self, device) - } - - fn handle_threaded(&self, device: &Device) -> IrqReturn { - T::handle_threaded(self, device) - } -} - -impl ThreadedHandler for Box { - fn handle(&self, device: &Device) -> ThreadedIrqReturn { - T::handle(self, device) - } - - fn handle_threaded(&self, device: &Device) -> IrqReturn { - T::handle_threaded(self, device) - } + fn handle_threaded(&self) -> IrqReturn; } /// A registration of a threaded IRQ handler for a given IRQ line. @@ -339,13 +296,20 @@ fn handle_threaded(&self, device: &Device) -> IrqReturn { /// [`Mutex`](kernel::sync::Mutex) to provide interior mutability. /// /// ``` -/// use kernel::device::{Bound, Device}; -/// use kernel::irq::{ -/// self, Flags, IrqRequest, IrqReturn, ThreadedHandler, ThreadedIrqReturn, -/// ThreadedRegistration, +/// use core::pin::Pin; +/// use kernel::{ +/// irq::{ +/// self, +/// Flags, +/// IrqRequest, +/// IrqReturn, +/// ThreadedHandler, +/// ThreadedIrqReturn, +/// ThreadedRegistration, +/// }, +/// prelude::*, +/// sync::Mutex, /// }; -/// use kernel::prelude::*; -/// use kernel::sync::{Arc, Mutex}; /// /// // Declare a struct that will be passed in when the interrupt fires. The u32 /// // merely serves as an example of some internal data. @@ -363,7 +327,7 @@ fn handle_threaded(&self, device: &Device) -> IrqReturn { /// // This will run (in a separate kthread) if and only if /// // [`ThreadedHandler::handle`] returns [`WakeThread`], which it does by /// // default. -/// fn handle_threaded(&self, _dev: &Device) -> IrqReturn { +/// fn handle_threaded(&self) -> IrqReturn { /// let mut data = self.value.lock(); /// *data += 1; /// IrqReturn::Handled @@ -375,13 +339,21 @@ fn handle_threaded(&self, device: &Device) -> IrqReturn { /// // This is executing in process context and assumes that `request` was /// // previously acquired from a device. /// fn register_threaded_irq( -/// handler: impl PinInit, /// request: IrqRequest<'_>, -/// ) -> Result>> { -/// let registration = -/// ThreadedRegistration::new(request, Flags::SHARED, c"my_device", handler); +/// ) -> Result>>> { +/// // SAFETY: The returned Registration is not leaked. +/// let registration = unsafe { +/// ThreadedRegistration::new( +/// request, +/// Flags::SHARED, +/// c"my_device", +/// try_pin_init!(Data { +/// value <- kernel::new_mutex!(0), +/// }? Error), +/// ) +/// }; /// -/// let registration = Arc::pin_init(registration, GFP_KERNEL)?; +/// let registration = KBox::pin_init(registration, GFP_KERNEL)?; /// /// { /// // The data can be accessed from process context too. @@ -396,11 +368,11 @@ fn handle_threaded(&self, device: &Device) -> IrqReturn { /// /// # Invariants /// -/// * We own an irq handler whose cookie is a pointer to `Self`. -#[pin_data] -pub struct ThreadedRegistration { - #[pin] - inner: Devres, +/// * We own an irq handler registered via `request_threaded_irq` whose cookie is a pointer to +/// `Self`. +#[pin_data(PinnedDrop)] +pub struct ThreadedRegistration<'a, T: ThreadedHandler> { + request: IrqRequest<'a>, #[pin] handler: T, @@ -411,45 +383,47 @@ pub struct ThreadedRegistration { _pin: PhantomPinned, } -impl ThreadedRegistration { +impl<'a, T: ThreadedHandler> ThreadedRegistration<'a, T> { /// Registers the IRQ handler with the system for the given IRQ number. - pub fn new<'a>( + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the returned [`ThreadedRegistration`] or otherwise prevent + /// its [`Drop`] implementation from running. + pub unsafe fn new( request: IrqRequest<'a>, flags: Flags, name: &'static CStr, handler: impl PinInit + 'a, - ) -> impl PinInit + 'a { + ) -> impl PinInit + 'a + where + T: 'a, + { + // INVARIANT: If initialization completes successfully, we own an IRQ handler registered + // via `request_threaded_irq` whose cookie is a pointer to `Self`. try_pin_init!(&this in Self { handler <- handler, - inner <- Devres::new( - request.dev, - try_pin_init!(RegistrationInner { - // INVARIANT: `this` is a valid pointer to the `ThreadedRegistration` instance. - cookie: this.as_ptr().cast::(), - irq: { - // SAFETY: - // - The callbacks are valid for use with request_threaded_irq. - // - If this succeeds, the slot is guaranteed to be valid until the - // destructor of Self runs, which will deregister the callbacks - // before the memory location becomes invalid. - // - When request_threaded_irq is called, everything that the two callbacks - // will touch has already been initialized, so it's safe for the - // callbacks to be called immediately. - to_result(unsafe { - bindings::request_threaded_irq( - request.irq, - Some(handle_threaded_irq_callback::), - Some(thread_fn_callback::), - flags.into_inner(), - name.as_char_ptr(), - this.as_ptr().cast::(), - ) - })?; - request.irq - } - }) - ), + request, _pin: PhantomPinned, + _: { + // SAFETY: + // - The callbacks are valid for use with request_threaded_irq. + // - If this succeeds, the slot is guaranteed to be valid until the destructor of + // Self runs, which will deregister the callbacks before the memory location + // becomes invalid. + // - All fields are already initialized, so it's safe for the callbacks to be + // called immediately. + to_result(unsafe { + bindings::request_threaded_irq( + request.irq, + Some(handle_threaded_irq_callback::), + Some(thread_fn_callback::), + flags.into_inner(), + name.as_char_ptr(), + this.as_ptr().cast::(), + ) + })?; + }, }) } @@ -459,19 +433,25 @@ pub fn handler(&self) -> &T { } /// Wait for pending IRQ handlers on other CPUs. - /// - /// This will attempt to access the inner [`Devres`] container. - pub fn try_synchronize(&self) -> Result { - let inner = self.inner.try_access().ok_or(ENODEV)?; - inner.synchronize(); - Ok(()) + #[inline] + pub fn synchronize(&self) { + // SAFETY: `self.request.irq` is a valid registered IRQ number (type invariant). + unsafe { bindings::synchronize_irq(self.request.irq) }; } +} - /// Wait for pending IRQ handlers on other CPUs. - pub fn synchronize(&self, dev: &Device) -> Result { - let inner = self.inner.access(dev)?; - inner.synchronize(); - Ok(()) +#[pinned_drop] +impl PinnedDrop for ThreadedRegistration<'_, T> { + fn drop(self: Pin<&mut Self>) { + // SAFETY: The cookie was set to a pointer to `Self` in `ThreadedRegistration::new()`. This + // blocks until all in-flight handlers complete, so no references to `self` remain after + // this returns. + unsafe { + bindings::free_irq( + self.request.irq, + core::ptr::from_mut::(self.get_unchecked_mut()).cast::(), + ) + }; } } @@ -482,24 +462,22 @@ pub fn synchronize(&self, dev: &Device) -> Result { _irq: i32, ptr: *mut c_void, ) -> c_uint { - // SAFETY: `ptr` is a pointer to `ThreadedRegistration` set in `ThreadedRegistration::new` - let registration = unsafe { &*(ptr as *const ThreadedRegistration) }; - // SAFETY: The irq callback is removed before the device is unbound, so the fact that the irq - // callback is running implies that the device has not yet been unbound. - let device = unsafe { registration.inner.device().as_bound() }; + let ptr = ptr.cast_const().cast::>(); + // SAFETY: `ptr` is a pointer to `ThreadedRegistration<'_, T>` set in + // `ThreadedRegistration::new()`. + let registration = unsafe { &*ptr }; - T::handle(®istration.handler, device) as c_uint + T::handle(®istration.handler) as c_uint } /// # Safety /// /// This function should be only used as the callback in `request_threaded_irq`. unsafe extern "C" fn thread_fn_callback(_irq: i32, ptr: *mut c_void) -> c_uint { - // SAFETY: `ptr` is a pointer to `ThreadedRegistration` set in `ThreadedRegistration::new` - let registration = unsafe { &*(ptr as *const ThreadedRegistration) }; - // SAFETY: The irq callback is removed before the device is unbound, so the fact that the irq - // callback is running implies that the device has not yet been unbound. - let device = unsafe { registration.inner.device().as_bound() }; + let ptr = ptr.cast_const().cast::>(); + // SAFETY: `ptr` is a pointer to `ThreadedRegistration<'_, T>` set in + // `ThreadedRegistration::new()`. + let registration = unsafe { &*ptr }; - T::handle_threaded(®istration.handler, device) as c_uint + T::handle_threaded(®istration.handler) as c_uint } diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 73fd046ee8ed..abb208518413 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -16,6 +16,9 @@ // Please see https://github.com/Rust-for-Linux/linux/issues/2 for details on // the unstable features in use. // +// Stable since Rust 1.87.0. +#![feature(unsigned_is_multiple_of)] +// // Stable since Rust 1.89.0. #![feature(generic_arg_infer)] // @@ -121,6 +124,8 @@ pub mod scatterlist; pub mod security; pub mod seq_file; +#[cfg(CONFIG_RUST_SERIAL_DEV_BUS_ABSTRACTIONS)] +pub mod serdev; pub mod sizes; #[cfg(CONFIG_SOC_BUS)] pub mod soc; diff --git a/rust/kernel/net/phy.rs b/rust/kernel/net/phy.rs index 8b7036b8fe48..956cda573ddb 100644 --- a/rust/kernel/net/phy.rs +++ b/rust/kernel/net/phy.rs @@ -804,62 +804,6 @@ const fn as_int(&self) -> u32 { /// } /// # } /// ``` -/// -/// This expands to the following code: -/// -/// ```ignore -/// use kernel::net::phy::{self, DeviceId}; -/// use kernel::prelude::*; -/// -/// struct Module { -/// _reg: ::kernel::net::phy::Registration, -/// } -/// -/// module! { -/// type: Module, -/// name: "rust_sample_phy", -/// authors: ["Rust for Linux Contributors"], -/// description: "Rust sample PHYs driver", -/// license: "GPL", -/// } -/// -/// struct PhySample; -/// -/// #[vtable] -/// impl phy::Driver for PhySample { -/// const NAME: &'static CStr = c"PhySample"; -/// const PHY_DEVICE_ID: phy::DeviceId = phy::DeviceId::new_with_exact_mask(0x00000001); -/// } -/// -/// const _: () = { -/// static mut DRIVERS: [::kernel::net::phy::DriverVTable; 1] = -/// [::kernel::net::phy::create_phy_driver::()]; -/// -/// impl ::kernel::Module for Module { -/// fn init(module: &'static ::kernel::ThisModule) -> Result { -/// let drivers = unsafe { &mut DRIVERS }; -/// let mut reg = ::kernel::net::phy::Registration::register( -/// module, -/// ::core::pin::Pin::static_mut(drivers), -/// )?; -/// Ok(Module { _reg: reg }) -/// } -/// } -/// }; -/// -/// const N: usize = 1; -/// -/// const TABLE: ::kernel::device_id::IdArray<::kernel::net::phy::DeviceId, (), N> = -/// ::kernel::device_id::IdArray::new_without_index([ -/// ::kernel::net::phy::DeviceId( -/// ::kernel::bindings::mdio_device_id { -/// phy_id: 0x00000001, -/// phy_id_mask: 0xffffffff, -/// }), -/// ]); -/// -/// ::kernel::module_device_table!("mdio", phydev, TABLE); -/// ``` #[macro_export] macro_rules! module_phy_driver { (@replace_expr $_t:tt $sub:expr) => {$sub}; @@ -869,12 +813,10 @@ macro_rules! module_phy_driver { }; (@device_table [$($dev:expr),+]) => { - const N: usize = $crate::module_phy_driver!(@count_devices $($dev),+); - - const TABLE: $crate::device_id::IdArray<$crate::net::phy::DeviceId, (), N> = - $crate::device_id::IdArray::new_without_index([ $(($dev,())),+, ]); - - $crate::module_device_table!("mdio", phydev, TABLE); + $crate::module_device_table!( + "mdio", $crate::net::phy::DeviceId, + TABLE, @none, [$($dev),+] + ); }; (drivers: [$($driver:ident),+ $(,)?], device_table: [$($dev:expr),+ $(,)?], $($f:tt)*) => { diff --git a/rust/kernel/of.rs b/rust/kernel/of.rs index 58b20c367f99..d0318f62afd7 100644 --- a/rust/kernel/of.rs +++ b/rust/kernel/of.rs @@ -25,10 +25,6 @@ unsafe impl RawDeviceId for DeviceId { // SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `data` field. unsafe impl RawDeviceIdIndex for DeviceId { const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::of_device_id, data); - - fn index(&self) -> usize { - self.0.data as usize - } } impl DeviceId { @@ -53,13 +49,7 @@ pub const fn new(compatible: &'static CStr) -> Self { /// Create an OF `IdTable` with an "alias" for modpost. #[macro_export] macro_rules! of_device_table { - ($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => { - const $table_name: $crate::device_id::IdArray< - $crate::of::DeviceId, - $id_info_type, - { $table_data.len() }, - > = $crate::device_id::IdArray::new($table_data); - - $crate::module_device_table!("of", $module_table_name, $table_name); + ($($tt:tt)*) => { + $crate::module_device_table!("of", $crate::of::DeviceId, $($tt)*); }; } diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs index 4def9ca1824c..aee6485dc58e 100644 --- a/rust/kernel/pci.rs +++ b/rust/kernel/pci.rs @@ -43,15 +43,16 @@ pub use self::io::{ Bar, ConfigSpace, - ConfigSpaceKind, ConfigSpaceSize, + DevresBar, Extended, Normal, // }; pub use self::irq::{ IrqType, IrqTypes, - IrqVector, // + IrqVector, + IrqVectorRegistration, // }; /// An adapter for the registration of PCI drivers. @@ -110,7 +111,11 @@ extern "C" fn probe_callback( // SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct pci_device_id` and // does not add additional invariants, so it's safe to transmute. let id = unsafe { &*id.cast::() }; - let info = T::ID_TABLE.info(id.index()); + + // SAFETY: `id` comes from `T::ID_TABLE` which is of type `IdArray<_, T::IdInfo>` or + // `pci_device_id_any` which has 0 as driver_data. It can also come from dynamic IDs, which + // will ensure that `driver_data` exists in `T::ID_TABLE`. + let info = unsafe { id.info_unchecked_opt::() }; from_result(|| { let data = T::probe(pdev, info); @@ -233,10 +238,6 @@ unsafe impl RawDeviceId for DeviceId { // SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_data` field. unsafe impl RawDeviceIdIndex for DeviceId { const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::pci_device_id, driver_data); - - fn index(&self) -> usize { - self.0.driver_data - } } /// `IdTable` type for PCI. @@ -245,14 +246,8 @@ fn index(&self) -> usize { /// Create a PCI `IdTable` with its alias for modpost. #[macro_export] macro_rules! pci_device_table { - ($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => { - const $table_name: $crate::device_id::IdArray< - $crate::pci::DeviceId, - $id_info_type, - { $table_data.len() }, - > = $crate::device_id::IdArray::new($table_data); - - $crate::module_device_table!("pci", $module_table_name, $table_name); + ($($tt:tt)*) => { + $crate::module_device_table!("pci", $crate::pci::DeviceId, $($tt)*); }; } @@ -267,7 +262,6 @@ macro_rules! pci_device_table { /// /// kernel::pci_device_table!( /// PCI_TABLE, -/// MODULE_PCI_TABLE, /// ::IdInfo, /// [ /// ( @@ -284,7 +278,7 @@ macro_rules! pci_device_table { /// /// fn probe<'bound>( /// _pdev: &'bound pci::Device>, -/// _id_info: &'bound Self::IdInfo, +/// _id_info: Option<&'bound Self::IdInfo>, /// ) -> impl PinInit, Error> + 'bound { /// Err(ENODEV) /// } @@ -313,7 +307,7 @@ pub trait Driver { /// attempt to initialize the device here. fn probe<'bound>( dev: &'bound Device>, - id_info: &'bound Self::IdInfo, + id_info: Option<&'bound Self::IdInfo>, ) -> impl PinInit, Error> + 'bound; /// PCI driver unbind. @@ -482,11 +476,13 @@ impl<'a> crate::dma::Device<'a> for Device> {} // SAFETY: Instances of `Device` are always reference-counted. unsafe impl crate::sync::aref::AlwaysRefCounted for Device { + #[inline] fn inc_ref(&self) { // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. unsafe { bindings::pci_dev_get(self.as_raw()) }; } + #[inline] unsafe fn dec_ref(obj: NonNull) { // SAFETY: The safety requirements guarantee that the refcount is non-zero. unsafe { bindings::pci_dev_put(obj.cast().as_ptr()) } diff --git a/rust/kernel/pci/io.rs b/rust/kernel/pci/io.rs index 0461e01aaa20..953e16735c6e 100644 --- a/rust/kernel/pci/io.rs +++ b/rust/kernel/pci/io.rs @@ -6,19 +6,22 @@ use crate::{ bindings, device, - devres::Devres, + devres::DevresLt, io::{ - Io, + IoBackend, + IoBase, IoCapable, - IoKnownSize, Mmio, - MmioRaw, // + MmioBackend, + MmioRaw, + Region, // }, - prelude::*, // -}; -use core::{ - marker::PhantomData, - ops::Deref, // + prelude::*, + ptr::KnownSize, + types::{ + CovariantForLt, + ForLt, // + }, // }; /// Represents the size of a PCI configuration space. @@ -46,68 +49,95 @@ pub const fn into_raw(self) -> usize { } } -/// Marker type for normal (256-byte) PCI configuration space. -pub struct Normal; +/// Alias for normal (256-byte) PCI configuration space. +pub type Normal = Region<256>; -/// Marker type for extended (4096-byte) PCIe configuration space. -pub struct Extended; +/// Alias for extended (4096-byte) PCIe configuration space. +pub type Extended = Region<4096>; -/// Trait for PCI configuration space size markers. -/// -/// This trait is implemented by [`Normal`] and [`Extended`] to provide -/// compile-time knowledge of the configuration space size. -pub trait ConfigSpaceKind { - /// The size of this configuration space in bytes. - const SIZE: usize; -} - -impl ConfigSpaceKind for Normal { - const SIZE: usize = 256; -} - -impl ConfigSpaceKind for Extended { - const SIZE: usize = 4096; -} - -/// The PCI configuration space of a device. +/// A view of PCI configuration space of a device. /// /// Provides typed read and write accessors for configuration registers /// using the standard `pci_read_config_*` and `pci_write_config_*` helpers. /// -/// The generic parameter `S` indicates the maximum size of the configuration space. -/// Use [`Normal`] for 256-byte legacy configuration space or [`Extended`] for -/// 4096-byte PCIe extended configuration space (default). -pub struct ConfigSpace<'a, S: ConfigSpaceKind = Extended> { +/// The generic parameter `T` is the type of the view. The full configuration space is also a +/// special type of view; in such cases, `T` can be [`Normal`] for 256-byte legacy configuration +/// space or [`Extended`] for 4096-byte PCIe extended configuration space (default). +/// +/// # Invariants +/// +/// `ptr` is aligned and range `ptr..ptr + KnownSize::size(ptr)` is within +/// `0..pdev.cfg_size().into_raw()`. +pub struct ConfigSpace<'a, T: ?Sized = Extended> { pub(crate) pdev: &'a Device, - _marker: PhantomData, + ptr: *mut T, +} + +impl Copy for ConfigSpace<'_, T> {} +impl Clone for ConfigSpace<'_, T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +// SAFETY: `ConfigSpace<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl Send for ConfigSpace<'_, T> {} + +// SAFETY: `ConfigSpace<'_, T>` is conceptually `&T` but in I/O memory. +unsafe impl Sync for ConfigSpace<'_, T> {} + +/// I/O Backend for PCI configuration space. +pub struct ConfigSpaceBackend; + +impl IoBackend for ConfigSpaceBackend { + type View<'a, T: ?Sized + KnownSize> = ConfigSpace<'a, T>; + + #[inline] + fn as_ptr<'a, T: ?Sized + KnownSize>(view: ConfigSpace<'a, T>) -> *mut T { + view.ptr + } + + #[inline] + unsafe fn project_view<'a, T: ?Sized + KnownSize, U: ?Sized + KnownSize>( + view: Self::View<'a, T>, + ptr: *mut U, + ) -> Self::View<'a, U> { + // INVARIANT: Per safety requirement. + ConfigSpace { + pdev: view.pdev, + ptr, + } + } } /// Implements [`IoCapable`] on [`ConfigSpace`] for `$ty` using `$read_fn` and `$write_fn`. macro_rules! impl_config_space_io_capable { ($ty:ty, $read_fn:ident, $write_fn:ident) => { - impl<'a, S: ConfigSpaceKind> IoCapable<$ty> for ConfigSpace<'a, S> { - unsafe fn io_read(&self, address: usize) -> $ty { + impl IoCapable<$ty> for ConfigSpaceBackend { + fn io_read(view: ConfigSpace<'_, $ty>) -> $ty { + // CAST: The offset is cast to `i32` because the C functions expect a 32-bit + // signed offset parameter. PCI configuration space size is at most 4096 bytes, + // so the value always fits within `i32` without truncation or sign change. + let addr = view.ptr.addr() as i32; + let mut val: $ty = 0; // Return value from C function is ignored in infallible accessors. - let _ret = - // SAFETY: By the type invariant `self.pdev` is a valid address. - // CAST: The offset is cast to `i32` because the C functions expect a 32-bit - // signed offset parameter. PCI configuration space size is at most 4096 bytes, - // so the value always fits within `i32` without truncation or sign change. - unsafe { bindings::$read_fn(self.pdev.as_raw(), address as i32, &mut val) }; - + // SAFETY: By the type invariant `pdev` is a valid address. + let _ = unsafe { bindings::$read_fn(view.pdev.as_raw(), addr, &mut val) }; val } - unsafe fn io_write(&self, value: $ty, address: usize) { + fn io_write(view: ConfigSpace<'_, $ty>, value: $ty) { + // CAST: The offset is cast to `i32` because the C functions expect a 32-bit + // signed offset parameter. PCI configuration space size is at most 4096 bytes, + // so the value always fits within `i32` without truncation or sign change. + let addr = view.ptr.addr() as i32; + // Return value from C function is ignored in infallible accessors. - let _ret = - // SAFETY: By the type invariant `self.pdev` is a valid address. - // CAST: The offset is cast to `i32` because the C functions expect a 32-bit - // signed offset parameter. PCI configuration space size is at most 4096 bytes, - // so the value always fits within `i32` without truncation or sign change. - unsafe { bindings::$write_fn(self.pdev.as_raw(), address as i32, value) }; + // SAFETY: By the type invariant `pdev` is a valid address. + let _ = unsafe { bindings::$write_fn(view.pdev.as_raw(), addr, value) }; } } }; @@ -118,24 +148,16 @@ unsafe fn io_write(&self, value: $ty, address: usize) { impl_config_space_io_capable!(u16, pci_read_config_word, pci_write_config_word); impl_config_space_io_capable!(u32, pci_read_config_dword, pci_write_config_dword); -impl<'a, S: ConfigSpaceKind> Io for ConfigSpace<'a, S> { - /// Returns the base address of the I/O region. It is always 0 for configuration space. - #[inline] - fn addr(&self) -> usize { - 0 - } +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for ConfigSpace<'a, T> { + type Backend = ConfigSpaceBackend; + type Target = T; - /// Returns the maximum size of the configuration space. #[inline] - fn maxsize(&self) -> usize { - self.pdev.cfg_size().into_raw() + fn as_view(self) -> ConfigSpace<'a, T> { + self } } -impl<'a, S: ConfigSpaceKind> IoKnownSize for ConfigSpace<'a, S> { - const MIN_SIZE: usize = S::SIZE; -} - /// A PCI BAR to perform I/O-Operations on. /// /// I/O backend assumes that the device is little-endian and will automatically @@ -147,10 +169,23 @@ impl<'a, S: ConfigSpaceKind> IoKnownSize for ConfigSpace<'a, S> { /// memory mapped PCI BAR and its size. pub struct Bar<'a, const SIZE: usize = 0> { pdev: &'a Device, - io: MmioRaw, + io: MmioRaw>, num: i32, } +impl ForLt for Bar<'static, SIZE> { + type Of<'a> = Bar<'a, SIZE>; +} + +// SAFETY: `Bar<'a, SIZE>` is covariant over `'a`; it holds `&'a Device`, +// which is covariant. +unsafe impl CovariantForLt for Bar<'static, SIZE> {} + +/// A device-managed PCI BAR mapping. +/// +/// See [`Bar::into_devres`]. +pub type DevresBar = DevresLt>; + impl<'a, const SIZE: usize> Bar<'a, SIZE> { pub(super) fn new( pdev: &'a Device, @@ -187,7 +222,7 @@ pub(super) fn new( return Err(ENOMEM); } - let io = match MmioRaw::new(ioptr, len as usize) { + let io = match MmioRaw::new_region(ioptr, len as usize) { Ok(io) => io, Err(err) => { // SAFETY: @@ -223,15 +258,13 @@ fn release(&self) { /// Consume the `Bar` and register it as a device-managed resource. /// - /// The returned `Devres>` can outlive the original lifetime `'a`. Access - /// to the BAR is revoked when the device is unbound. - pub fn into_devres(self) -> Result>> { - // SAFETY: Casting to `'static` is sound because `Devres` guarantees the `Bar` does not - // actually outlive the device -- access is revoked and the resource is released when the - // device is unbound. - let bar: Bar<'static, SIZE> = unsafe { core::mem::transmute(self) }; - let pdev = bar.pdev; - Devres::new(pdev.as_ref(), bar) + /// The returned [`DevresBar`] can outlive the original borrow and be stored in driver data. + /// Access to the BAR is revoked automatically when the device is unbound. + pub fn into_devres(self) -> Result> { + let pdev = self.pdev; + // SAFETY: `Bar` only holds a reference to the device and an I/O mapping, both of which + // remain valid for the device's full bound scope, not just for `'a`. + unsafe { DevresLt::new(pdev.as_ref(), self) } } } @@ -249,12 +282,14 @@ fn drop(&mut self) { } } -impl Deref for Bar<'_, SIZE> { - type Target = Mmio; +impl<'a, const SIZE: usize> IoBase<'a> for &'a Bar<'_, SIZE> { + type Backend = MmioBackend; + type Target = crate::io::Region; - fn deref(&self) -> &Self::Target { + #[inline] + fn as_view(self) -> Mmio<'a, Self::Target> { // SAFETY: By the type invariant of `Self`, the MMIO range in `self.io` is properly mapped. - unsafe { Mmio::from_raw(&self.io) } + unsafe { Mmio::from_raw(self.io) } } } @@ -289,23 +324,25 @@ pub fn cfg_size(&self) -> ConfigSpaceSize { } } - /// Return an initialized normal (256-byte) config space object. + /// Return a view of the normal (256-byte) config space. pub fn config_space<'a>(&'a self) -> ConfigSpace<'a, Normal> { + // INVARIANT: null is aligned and the range is within config space. ConfigSpace { pdev: self, - _marker: PhantomData, + ptr: Normal::ptr_from_raw_parts_mut(core::ptr::null_mut(), self.cfg_size().into_raw()), } } - /// Return an initialized extended (4096-byte) config space object. + /// Return a view of the extended (4096-byte) config space. pub fn config_space_extended<'a>(&'a self) -> Result> { if self.cfg_size() != ConfigSpaceSize::Extended { return Err(EINVAL); } + // INVARIANT: null is aligned and we just checked the `cfg_size`. Ok(ConfigSpace { pdev: self, - _marker: PhantomData, + ptr: Extended::ptr_from_raw_parts_mut(core::ptr::null_mut(), 4096), }) } } diff --git a/rust/kernel/pci/irq.rs b/rust/kernel/pci/irq.rs index d9230e105541..6741046ec1c0 100644 --- a/rust/kernel/pci/irq.rs +++ b/rust/kernel/pci/irq.rs @@ -7,17 +7,11 @@ bindings, device, device::Bound, - devres, error::to_result, - irq::{ - self, - IrqRequest, // - }, - prelude::*, - str::CStr, - sync::aref::ARef, // + irq::IrqRequest, + prelude::*, // }; -use core::ops::RangeInclusive; +use core::num::NonZero; /// IRQ type flags for PCI interrupt allocation. #[derive(Debug, Clone, Copy)] @@ -39,6 +33,16 @@ const fn as_raw(self) -> u32 { IrqType::MsiX => bindings::PCI_IRQ_MSIX, } } + + /// Construct from raw value. + #[inline] + const fn from_raw(raw: u32) -> Self { + match raw { + bindings::PCI_IRQ_MSIX => IrqType::MsiX, + bindings::PCI_IRQ_MSI => IrqType::Msi, + _ => IrqType::Intx, + } + } } /// Set of IRQ types that can be used for PCI interrupt allocation. @@ -71,148 +75,113 @@ const fn as_raw(self) -> u32 { } } -/// Represents an allocated IRQ vector for a specific PCI device. +/// A resolved IRQ vector from a PCI interrupt vector allocation. /// -/// This type ties an IRQ vector to the device it was allocated for, -/// ensuring the vector is only used with the correct device. -#[derive(Clone, Copy)] +/// Created by [`IrqVectorRegistration::index`]. Convert to [`IrqRequest`] via [`From`] to register +/// a handler with [`irq::Registration::new`](crate::irq::Registration::new). pub struct IrqVector<'a> { - dev: &'a Device, - index: u32, + request: IrqRequest<'a>, + reg: &'a IrqVectorRegistration<'a>, } impl<'a> IrqVector<'a> { - /// Creates a new [`IrqVector`] for the given device and index. + /// Creates a new [`IrqVector`] with an already resolved [`IrqRequest`]. /// /// # Safety /// - /// - `index` must be a valid IRQ vector index for `dev`. - /// - `dev` must point to a [`Device`] that has successfully allocated IRQ vectors. - unsafe fn new(dev: &'a Device, index: u32) -> Self { - Self { dev, index } + /// `request` must have been resolved from `reg`. + #[inline] + unsafe fn new(request: IrqRequest<'a>, reg: &'a IrqVectorRegistration<'a>) -> Self { + Self { request, reg } } - /// Returns the raw vector index. - fn index(&self) -> u32 { - self.index + /// Returns the [`IrqVectorRegistration`] this vector was derived from. + #[inline] + pub fn vectors(&self) -> &'a IrqVectorRegistration<'a> { + self.reg + } + + /// Returns the interrupt type the PCI core selected for this vector's allocation. + #[inline] + pub fn irq_type(&self) -> IrqType { + self.reg.irq_type() } } -impl<'a> TryInto> for IrqVector<'a> { - type Error = Error; - - fn try_into(self) -> Result> { - // SAFETY: `self.as_raw` returns a valid pointer to a `struct pci_dev`. - let irq = unsafe { bindings::pci_irq_vector(self.dev.as_raw(), self.index()) }; - if irq < 0 { - return Err(crate::error::Error::from_errno(irq)); - } - // SAFETY: `irq` is guaranteed to be a valid IRQ number for `&self`. - Ok(unsafe { IrqRequest::new(self.dev.as_ref(), irq as u32) }) +impl<'a> From> for IrqRequest<'a> { + #[inline] + fn from(vector: IrqVector<'a>) -> Self { + vector.request } } -/// Represents an IRQ vector allocation for a PCI device. +/// An allocation of PCI interrupt vectors for a device. /// -/// This type ensures that IRQ vectors are properly allocated and freed by -/// tying the allocation to the lifetime of this registration object. +/// This type owns the vector allocation; dropping it frees the vectors. IRQ handlers borrow from +/// this registration and must be dropped before it is. /// /// # Invariants /// -/// The [`Device`] has successfully allocated IRQ vectors. -struct IrqVectorRegistration { - dev: ARef, +/// `dev` has an allocation of `len` interrupt vectors. +pub struct IrqVectorRegistration<'a> { + dev: &'a Device, + len: NonZero, } -impl IrqVectorRegistration { - /// Allocate and register IRQ vectors for the given PCI device. +impl<'a> IrqVectorRegistration<'a> { + /// Returns the number of allocated vectors. /// - /// Allocates IRQ vectors and registers them with devres for automatic cleanup. - /// Returns a range of valid IRQ vectors. - fn register<'a>( - dev: &'a Device, - min_vecs: u32, - max_vecs: u32, - irq_types: IrqTypes, - ) -> Result>> { - // SAFETY: - // - `dev.as_raw()` is guaranteed to be a valid pointer to a `struct pci_dev` - // by the type invariant of `Device`. - // - `pci_alloc_irq_vectors` internally validates all other parameters - // and returns error codes. - let ret = unsafe { - bindings::pci_alloc_irq_vectors(dev.as_raw(), min_vecs, max_vecs, irq_types.as_raw()) - }; + /// This is at least the `min_vecs` that [`Device::alloc_irq_vectors`] was asked for. + #[inline] + #[allow(clippy::len_without_is_empty)] + pub fn len(&self) -> usize { + self.len.get() + } - to_result(ret)?; - let count = ret as u32; + /// Returns the interrupt type the PCI core selected for this allocation. + #[inline] + pub fn irq_type(&self) -> IrqType { + // SAFETY: `self.dev.as_raw()` is a valid pointer to a `struct pci_dev`. + IrqType::from_raw(unsafe { bindings::pci_irq_type(self.dev.as_raw()) }) + } - // SAFETY: - // - `pci_alloc_irq_vectors` returns the number of allocated vectors on success. - // - Vectors are 0-based, so valid indices are [0, count-1]. - // - `pci_alloc_irq_vectors` guarantees `count >= min_vecs > 0`, so both `0` and - // `count - 1` are valid IRQ vector indices for `dev`. - let range = unsafe { IrqVector::new(dev, 0)..=IrqVector::new(dev, count - 1) }; + /// Returns the [`IrqVector`] at `index`. + /// + /// Returns [`EINVAL`] if the `index` is out of bounds for the length reported by + /// [`Self::len()`]. + #[inline] + pub fn index(&self, index: usize) -> Result> { + // SAFETY: `self.dev.as_raw()` is a valid pointer to a `struct pci_dev`. + let irq = unsafe { bindings::pci_irq_vector(self.dev.as_raw(), index as u32) }; + if irq < 0 { + return Err(Error::from_errno(irq)); + } - // INVARIANT: The IRQ vector allocation for `dev` above was successful. - let irq_vecs = Self { dev: dev.into() }; - devres::register(dev.as_ref(), irq_vecs, GFP_KERNEL)?; - - Ok(range) + // SAFETY: `irq` is a valid IRQ number for `self.dev`, resolved from this registration. + Ok(unsafe { IrqVector::new(IrqRequest::new(self.dev.as_ref(), irq as u32), self) }) } } -impl Drop for IrqVectorRegistration { +impl Drop for IrqVectorRegistration<'_> { + #[inline] fn drop(&mut self) { - // SAFETY: - // - By the type invariant, `self.dev.as_raw()` is a valid pointer to a `struct pci_dev`. - // - `self.dev` has successfully allocated IRQ vectors. + // SAFETY: By the type invariant, `self.dev.as_raw()` is a valid pointer to a + // `struct pci_dev` that has successfully allocated IRQ vectors. unsafe { bindings::pci_free_irq_vectors(self.dev.as_raw()) }; } } impl Device { - /// Returns a [`kernel::irq::Registration`] for the given IRQ vector. - pub fn request_irq<'a, T: crate::irq::Handler + 'static>( - &'a self, - vector: IrqVector<'a>, - flags: irq::Flags, - name: &'static CStr, - handler: impl PinInit + 'a, - ) -> impl PinInit, Error> + 'a { - pin_init::pin_init_scope(move || { - let request = vector.try_into()?; - - Ok(irq::Registration::::new(request, flags, name, handler)) - }) - } - - /// Returns a [`kernel::irq::ThreadedRegistration`] for the given IRQ vector. - pub fn request_threaded_irq<'a, T: crate::irq::ThreadedHandler + 'static>( - &'a self, - vector: IrqVector<'a>, - flags: irq::Flags, - name: &'static CStr, - handler: impl PinInit + 'a, - ) -> impl PinInit, Error> + 'a { - pin_init::pin_init_scope(move || { - let request = vector.try_into()?; - - Ok(irq::ThreadedRegistration::::new( - request, flags, name, handler, - )) - }) - } - - /// Allocate IRQ vectors for this PCI device with automatic cleanup. + /// Allocate IRQ vectors for this PCI device. /// /// Allocates between `min_vecs` and `max_vecs` interrupt vectors for the device. /// The allocation will use MSI-X, MSI, or INTx interrupts based on the `irq_types` /// parameter and hardware capabilities. When multiple types are specified, the kernel /// will try them in order of preference: MSI-X first, then MSI, then INTx interrupts. /// - /// The allocated vectors are automatically freed when the device is unbound, using the - /// devres (device resource management) system. + /// The allocated vectors are freed when the returned [`IrqVectorRegistration`] is dropped. + /// Use [`IrqVectorRegistration::index`] to obtain an [`IrqVector`] for a given vector + /// index. /// /// # Arguments /// @@ -222,8 +191,8 @@ pub fn request_threaded_irq<'a, T: crate::irq::ThreadedHandler + 'static>( /// /// # Returns /// - /// Returns a range of IRQ vectors that were successfully allocated, or an error if the - /// allocation fails or cannot meet the minimum requirement. + /// Returns the IRQ vector registration, or an error if `min_vecs` vectors cannot be + /// allocated. /// /// # Examples /// @@ -246,7 +215,20 @@ pub fn alloc_irq_vectors( min_vecs: u32, max_vecs: u32, irq_types: IrqTypes, - ) -> Result>> { - IrqVectorRegistration::register(self, min_vecs, max_vecs, irq_types) + ) -> Result> { + // SAFETY: + // - `self.as_raw()` is guaranteed to be a valid pointer to a `struct pci_dev` + // by the type invariant of `Device`. + // - `pci_alloc_irq_vectors` internally validates all other parameters + // and returns error codes. + let ret = unsafe { + bindings::pci_alloc_irq_vectors(self.as_raw(), min_vecs, max_vecs, irq_types.as_raw()) + }; + to_result(ret)?; + + let len = NonZero::new(ret as usize).ok_or(EINVAL)?; + + // INVARIANT: `pci_alloc_irq_vectors()` allocated `len` vectors for `self`. + Ok(IrqVectorRegistration { dev: self, len }) } } diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs index 5a5f4156d79b..ac0a012ae1bb 100644 --- a/rust/kernel/platform.rs +++ b/rust/kernel/platform.rs @@ -100,7 +100,8 @@ extern "C" fn probe_callback(pdev: *mut bindings::platform_device) -> kernel::ff // // INVARIANT: `pdev` is valid for the duration of `probe_callback()`. let pdev = unsafe { &*pdev.cast::>>() }; - let info = ::id_info(pdev.as_ref()); + // SAFETY: `pdev` matched data is of type `Self::IdInfo`. + let info = unsafe { ::id_info(pdev.as_ref()) }; from_result(|| { let data = T::probe(pdev, info); @@ -176,7 +177,6 @@ macro_rules! module_platform_driver { /// /// kernel::of_device_table!( /// OF_TABLE, -/// MODULE_OF_TABLE, /// ::IdInfo, /// [ /// (of::DeviceId::new(c"test,device"), ()) @@ -185,7 +185,6 @@ macro_rules! module_platform_driver { /// /// kernel::acpi_device_table!( /// ACPI_TABLE, -/// MODULE_ACPI_TABLE, /// ::IdInfo, /// [ /// (acpi::DeviceId::new(c"LNUXBEEF"), ()) @@ -340,22 +339,30 @@ macro_rules! define_irq_accessor_by_index { $handler_trait:ident ) => { $(#[$meta])* - pub fn $fn_name<'a, T: irq::$handler_trait + 'static>( + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the resulting registration or otherwise prevent its + /// [`Drop`] implementation from running. + pub unsafe fn $fn_name<'a, T: irq::$handler_trait + 'a>( &'a self, flags: irq::Flags, index: u32, name: &'static CStr, handler: impl PinInit + 'a, - ) -> impl PinInit, Error> + 'a { + ) -> impl PinInit, Error> + 'a { pin_init::pin_init_scope(move || { let request = self.$request_fn(index)?; - Ok(irq::$reg_type::::new( - request, - flags, - name, - handler, - )) + // SAFETY: Caller guarantees the Registration will not be leaked. + Ok(unsafe { + irq::$reg_type::::new( + request, + flags, + name, + handler, + ) + }) }) } }; @@ -369,22 +376,30 @@ macro_rules! define_irq_accessor_by_name { $handler_trait:ident ) => { $(#[$meta])* - pub fn $fn_name<'a, T: irq::$handler_trait + 'static>( + /// + /// # Safety + /// + /// Callers must not `mem::forget()` the resulting registration or otherwise prevent its + /// [`Drop`] implementation from running. + pub unsafe fn $fn_name<'a, T: irq::$handler_trait + 'a>( &'a self, flags: irq::Flags, irq_name: &'a CStr, name: &'static CStr, handler: impl PinInit + 'a, - ) -> impl PinInit, Error> + 'a { + ) -> impl PinInit, Error> + 'a { pin_init::pin_init_scope(move || { let request = self.$request_fn(irq_name)?; - Ok(irq::$reg_type::::new( - request, - flags, - name, - handler, - )) + // SAFETY: Caller guarantees the Registration will not be leaked. + Ok(unsafe { + irq::$reg_type::::new( + request, + flags, + name, + handler, + ) + }) }) } }; diff --git a/rust/kernel/ptr.rs b/rust/kernel/ptr.rs index 3f3e529e9f58..82acb531b17b 100644 --- a/rust/kernel/ptr.rs +++ b/rust/kernel/ptr.rs @@ -235,11 +235,20 @@ fn align_up(self, alignment: Alignment) -> Option { /// /// This is a generalization of [`size_of`] that works for dynamically sized types. pub trait KnownSize { + /// Minimum size of this type known at compile-time. + const MIN_SIZE: usize; + + /// Minimum alignment of this type known at compile-time. + const MIN_ALIGN: Alignment; + /// Get the size of an object of this type in bytes, with the metadata of the given pointer. fn size(p: *const Self) -> usize; } impl KnownSize for T { + const MIN_SIZE: usize = size_of::(); + const MIN_ALIGN: Alignment = Alignment::of::(); + #[inline(always)] fn size(_: *const Self) -> usize { size_of::() @@ -247,6 +256,9 @@ fn size(_: *const Self) -> usize { } impl KnownSize for [T] { + const MIN_SIZE: usize = 0; + const MIN_ALIGN: Alignment = Alignment::of::(); + #[inline(always)] fn size(p: *const Self) -> usize { p.len() * size_of::() diff --git a/rust/kernel/serdev.rs b/rust/kernel/serdev.rs new file mode 100644 index 000000000000..17ca504b7f8d --- /dev/null +++ b/rust/kernel/serdev.rs @@ -0,0 +1,604 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Abstractions for the serial device bus. +//! +//! C header: [`include/linux/serdev.h`](srctree/include/linux/serdev.h) + +use crate::{ + acpi, + device, + driver, + error::{ + from_result, + to_result, + VTABLE_DEFAULT_ERROR, // + }, + new_mutex, + of, + prelude::*, + sync::{ + aref::AlwaysRefCounted, + Mutex, // + }, + time::Jiffies, + types::{ + Opaque, + ScopeGuard, // + }, // +}; + +use core::{ + cell::UnsafeCell, + marker::PhantomData, + mem::{offset_of, MaybeUninit}, + ptr::NonNull, // +}; + +/// Parity bit to use with a serial device. +#[repr(u32)] +pub enum Parity { + /// No parity bit. + None = bindings::serdev_parity_SERDEV_PARITY_NONE, + /// Even partiy. + Even = bindings::serdev_parity_SERDEV_PARITY_EVEN, + /// Odd parity. + Odd = bindings::serdev_parity_SERDEV_PARITY_ODD, +} + +/// An adapter for the registration of serial device bus device drivers. +pub struct Adapter(T); + +// SAFETY: +// - `bindings::serdev_device_driver` is a C type declared as `repr(C)`. +// - `PrivateData<'bound, T>` is the type of the driver's device private data. +// - `struct serdev_device_driver` embeds a `struct device_driver`. +// - `DEVICE_DRIVER_OFFSET` is the correct byte offset to the embedded `struct device_driver`. +unsafe impl driver::DriverLayout for Adapter { + type DriverType = bindings::serdev_device_driver; + type DriverData<'bound> = PrivateData<'bound, T>; + const DEVICE_DRIVER_OFFSET: usize = core::mem::offset_of!(Self::DriverType, driver); +} + +// SAFETY: A call to `unregister` for a given instance of `DriverType` is guaranteed to be valid if +// a preceding call to `register` has been successful. +unsafe impl driver::RegistrationOps for Adapter { + unsafe fn register( + sdrv: &Opaque, + name: &'static CStr, + module: &'static ThisModule, + ) -> Result { + let of_table = match T::OF_ID_TABLE { + Some(table) => table.as_ptr(), + None => core::ptr::null(), + }; + + let acpi_table = match T::ACPI_ID_TABLE { + Some(table) => table.as_ptr(), + None => core::ptr::null(), + }; + + // SAFETY: It's safe to set the fields of `struct serdev_device_driver` on initialization. + unsafe { + (*sdrv.get()).driver.name = name.as_char_ptr(); + (*sdrv.get()).probe = Some(Self::probe_callback); + (*sdrv.get()).remove = Some(Self::remove_callback); + (*sdrv.get()).driver.of_match_table = of_table; + (*sdrv.get()).driver.acpi_match_table = acpi_table; + } + + // SAFETY: `sdrv` is guaranteed to be a valid `DriverType`. + to_result(unsafe { bindings::__serdev_device_driver_register(sdrv.get(), module.as_ptr()) }) + } + + unsafe fn unregister(sdrv: &Opaque) { + // SAFETY: `sdrv` is guaranteed to be a valid `DriverType`. + unsafe { bindings::serdev_device_driver_unregister(sdrv.get()) }; + } +} + +#[doc(hidden)] +#[pin_data(PinnedDrop)] +pub struct PrivateData<'bound, T: Driver> { + sdev: &'bound Device, + #[pin] + driver: UnsafeCell>>, + open: UnsafeCell, + /// Whether `receive_buf_callback` is allowed to call `Driver::receive`. + /// + /// If locked, the receive_buf_callback will be blocked on data reception. + /// This is the case while the driver is being probed or while [`PrivateData`] is being dropped. + /// This is necessary, because we need to open the serdev device before the driver has been + /// probed in order to allow it to be configured, which allows `receive_buf_callback` to be + /// called. Thus we need to block data until probe completes and the driver data becomes + /// initialized. + /// + /// If unlocked and true, the receive_buf_callback will forward the data to + /// `Driver::receive`. This is the normal state of operation. + /// + /// If unlocked and false, the receive_buf_callback will throw away the data. + /// This is only the case, if the serdev device is open and + /// - the driver returned an error in probe + /// or + /// - the driver data already has been dropped, because it was unbound. + #[pin] + active: Mutex, +} + +#[pinned_drop] +impl PinnedDrop for PrivateData<'_, T> { + fn drop(self: Pin<&mut Self>) { + let mut active = self.active.lock(); + if *active { + // SAFETY: + // - We have exclusive access to `self.driver`. + // - `self.driver` is guaranteed to be initialized. + unsafe { (*self.driver.get()).assume_init_drop() }; + *active = false; + } + drop(active); + + // SAFETY: We have exclusive access to `self.open`. + if unsafe { *self.open.get() } { + // SAFETY: `self.sdev.as_raw()` is guaranteed to be a pointer to a valid + // `struct serdev_device`. + unsafe { bindings::serdev_device_close(self.sdev.as_raw()) }; + } + } +} + +impl Adapter { + const OPS: &'static bindings::serdev_device_ops = &bindings::serdev_device_ops { + receive_buf: if T::HAS_RECEIVE { + Some(Self::receive_buf_callback) + } else { + None + }, + write_wakeup: Some(bindings::serdev_device_write_wakeup), + }; + + extern "C" fn probe_callback(sdev: *mut bindings::serdev_device) -> kernel::ffi::c_int { + // SAFETY: The serial device bus only ever calls the probe callback with a valid pointer to + // a `struct serdev_device`. + // + // INVARIANT: `sdev` is valid for the duration of `probe_callback()`. + let sdev = unsafe { &*sdev.cast::>>() }; + // SAFETY: `sdev` matched data is of type `Self::IdInfo`. + let info = unsafe { ::id_info(sdev.as_ref()) }; + + from_result(|| { + sdev.as_ref().set_drvdata(try_pin_init!(PrivateData:: { + sdev: &**sdev, + driver: MaybeUninit::>::zeroed().into(), + open: false.into(), + active <- new_mutex!(false), + }))?; + // SAFETY: We just set drvdata to `PrivateData<'_, T>`. + let private_data = unsafe { sdev.as_ref().drvdata_borrow::>() }; + let private_data = ScopeGuard::new_with_data(private_data, |_| { + // SAFETY: We just set drvdata to `PrivateData<'_, T>`. + drop(unsafe { sdev.as_ref().drvdata_obtain::>() }); + }); + let mut active = private_data.active.lock(); + + // SAFETY: `sdev.as_raw()` is guaranteed to be a valid pointer to `serdev_device`. + unsafe { bindings::serdev_device_set_client_ops(sdev.as_raw(), Self::OPS) }; + + // SAFETY: The serial device bus only ever calls the probe callback with a valid pointer + // to a `serdev_device`. + to_result(unsafe { bindings::serdev_device_open(sdev.as_raw()) })?; + + // SAFETY: We have exclusive access to `private_data.open`. + unsafe { *private_data.open.get() = true }; + + let data = T::probe(sdev, info); + + // SAFETY: We have exclusive access to `private_data.driver`. + let driver = unsafe { &mut *private_data.driver.get() }; + // SAFETY: + // - `driver.as_mut_ptr()` is a valid pointer to uninitialized data. + // - `private_data.driver` is pinned. + let result = unsafe { pin_init::raw_try_init(driver.as_mut_ptr(), data) }; + + *active = result.is_ok(); + + drop(active); + + result.map(|()| { + private_data.dismiss(); + 0 + }) + }) + } + + extern "C" fn remove_callback(sdev: *mut bindings::serdev_device) { + // SAFETY: The serial device bus only ever calls the remove callback with a valid pointer + // to a `struct serdev_device`. + // + // INVARIANT: `sdev` is valid for the duration of `remove_callback()`. + let sdev = unsafe { &*sdev.cast::>>() }; + + // SAFETY: `remove_callback` is only ever called after a successful call to + // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called + // and stored a `Pin>>`. + let private_data = unsafe { sdev.as_ref().drvdata_borrow::>() }; + + // SAFETY: No one has exclusive access to `private_data.driver`. + let data = unsafe { &*private_data.driver.get() }; + // SAFETY: + // - `private_data.driver` is pinned. + // - `remove_callback` is only ever called after a successful call to `probe_callback`, + // hence it's guaranteed that `private_data.driver` was initialized. + let data_pinned = unsafe { Pin::new_unchecked(data.assume_init_ref()) }; + + T::unbind(sdev, data_pinned); + } + + extern "C" fn receive_buf_callback( + sdev: *mut bindings::serdev_device, + buf: *const u8, + length: usize, + ) -> usize { + // SAFETY: The serial device bus only ever calls the receive buf callback with a valid + // pointer to a `struct serdev_device`. + // + // INVARIANT: `sdev` is valid for the duration of `receive_buf_callback()`. + let sdev = unsafe { &*sdev.cast::>() }; + + // SAFETY: `receive_buf_callback` is only ever called after a successful call to + // `probe_callback`, hence it's guaranteed that `Device::set_drvdata()` has been called + // and stored a `Pin>>`. + let private_data = unsafe { sdev.as_ref().drvdata_borrow::>() }; + let active = private_data.active.lock(); + + if !*active { + return length; + } + + // SAFETY: No one has exclusive access to `private_data.driver`. + let data = unsafe { &*private_data.driver.get() }; + // SAFETY: + // - `private_data.driver` is pinned. + // - `receive_buf_callback` is only ever called after a successful call to `probe_callback`, + // hence it's guaranteed that `private_data.driver` was initialized. + let data_pinned = unsafe { Pin::new_unchecked(data.assume_init_ref()) }; + + // SAFETY: `buf` is guaranteed to be non-null and has the size of `length`. + let buf = unsafe { core::slice::from_raw_parts(buf, length) }; + + T::receive(sdev, data_pinned, buf) + } +} + +impl driver::Adapter for Adapter { + type IdInfo = T::IdInfo; + + fn of_id_table() -> Option> { + T::OF_ID_TABLE + } + + fn acpi_id_table() -> Option> { + T::ACPI_ID_TABLE + } +} + +/// Declares a kernel module that exposes a single serial device bus device driver. +/// +/// # Examples +/// +/// ```ignore +/// kernel::module_serdev_device_driver! { +/// type: MyDriver, +/// name: "Module name", +/// authors: ["Author name"], +/// description: "Description", +/// license: "GPL v2", +/// } +/// ``` +#[macro_export] +macro_rules! module_serdev_device_driver { + ($($f:tt)*) => { + $crate::module_driver!(, $crate::serdev::Adapter, { $($f)* }); + }; +} + +/// The serial device bus device driver trait. +/// +/// Drivers must implement this trait in order to get a serial device bus device driver registered. +/// +/// # Examples +/// +///``` +/// # use kernel::{ +/// acpi, +/// bindings, +/// device::{ +/// Bound, +/// Core, // +/// }, +/// of, +/// serdev, // +/// }; +/// +/// struct MyDriver; +/// +/// kernel::of_device_table!( +/// OF_TABLE, +/// ::IdInfo, +/// [ +/// (of::DeviceId::new(c"test,device"), ()) +/// ] +/// ); +/// +/// kernel::acpi_device_table!( +/// ACPI_TABLE, +/// ::IdInfo, +/// [ +/// (acpi::DeviceId::new(c"LNUXBEEF"), ()) +/// ] +/// ); +/// +/// #[vtable] +/// impl serdev::Driver for MyDriver { +/// type IdInfo = (); +/// type Data<'bound> = Self; +/// const OF_ID_TABLE: Option> = Some(&OF_TABLE); +/// const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); +/// +/// fn probe<'bound>( +/// sdev: &'bound serdev::Device>, +/// _id_info: Option<&'bound Self::IdInfo>, +/// ) -> impl PinInit, Error> + 'bound { +/// sdev.set_baudrate(115200); +/// sdev.write_all(b"Hello\n", 0)?; +/// Ok(MyDriver) +/// } +/// } +///``` +#[vtable] +pub trait Driver { + /// The type holding driver private data about each device id supported by the driver. + // TODO: Use associated_type_defaults once stabilized: + // + // ``` + // type IdInfo: 'static = (); + // ``` + type IdInfo: 'static; + + /// The type of the driver's bus device private data. + type Data<'bound>: Send + Sync + 'bound; + + /// The table of OF device ids supported by the driver. + const OF_ID_TABLE: Option> = None; + + /// The table of ACPI device ids supported by the driver. + const ACPI_ID_TABLE: Option> = None; + + /// Serial device bus device driver probe. + /// + /// Called when a new serial device bus device is added or discovered. + /// Implementers should attempt to initialize the device here. + fn probe<'bound>( + sdev: &'bound Device>, + id_info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit, Error> + 'bound; + + /// Serial device bus device driver unbind. + /// + /// Called when a [`Device`] is unbound from its bound [`Driver`]. Implementing this callback + /// is optional. + /// + /// This callback serves as a place for drivers to perform teardown operations that require a + /// `&Device` or `&Device` reference. For instance. + /// + /// Otherwise, release operations for driver resources should be performed in `Drop`. + fn unbind<'bound>(sdev: &'bound Device>, this: Pin<&Self::Data<'bound>>) { + let _ = (sdev, this); + } + + /// Serial device bus device data receive callback. + /// + /// Called when data got received from device. + /// + /// Returns the number of bytes accepted. + fn receive<'bound>( + sdev: &'bound Device, + this: Pin<&Self::Data<'bound>>, + data: &[u8], + ) -> usize { + let _ = (sdev, this, data); + build_error!(VTABLE_DEFAULT_ERROR) + } +} + +/// The serial device bus device representation. +/// +/// This structure represents the Rust abstraction for a C `struct serdev_device`. The +/// implementation abstracts the usage of an already existing C `struct serdev_device` within Rust +/// code that we get passed from the C side. +/// +/// # Invariants +/// +/// A [`Device`] instance represents a valid `struct serdev_device` created by the C portion of +/// the kernel. +#[repr(transparent)] +pub struct Device( + Opaque, + PhantomData, +); + +impl Device { + #[inline] + fn as_raw(&self) -> *mut bindings::serdev_device { + self.0.get() + } +} + +impl Device { + /// Set the baudrate in bits per second. + /// + /// Common baudrates are 115200, 9600, 19200, 57600, 4800. + /// + /// Use [`Device::write_flush`] before calling this if you have written data prior to this call. + #[inline] + pub fn set_baudrate(&self, speed: u32) -> Result<(), u32> { + // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + let ret = unsafe { bindings::serdev_device_set_baudrate(self.as_raw(), speed) }; + if ret == speed { + Ok(()) + } else { + Err(ret) + } + } + + /// Set if flow control should be enabled. + /// + /// Use [`Device::write_flush`] before calling this if you have written data prior to this call. + #[inline] + pub fn set_flow_control(&self, enable: bool) { + // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + unsafe { bindings::serdev_device_set_flow_control(self.as_raw(), enable) }; + } + + /// Set parity to use. + /// + /// Use [`Device::write_flush`] before calling this if you have written data prior to this call. + #[inline] + pub fn set_parity(&self, parity: Parity) -> Result { + // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + to_result(unsafe { bindings::serdev_device_set_parity(self.as_raw(), parity as u32) }) + } + + /// Write data to the serial device until the controller has accepted all the data or has + /// been interrupted by a timeout or signal. + /// + /// Note that any accepted data has only been buffered by the controller. Use + /// [`Device::wait_until_sent`] to make sure the controller write buffer has actually been + /// emptied. + /// + /// Use a timeout of 0 to wait indefinitely. + /// + /// Returns the number of bytes written (less than `data.len()` if interrupted). + /// [`kernel::error::code::ETIMEDOUT`] or [`kernel::error::code::ERESTARTSYS`] if interrupted + /// before any bytes were written. [`kernel::error::code::EINVAL`] if `data.len() > i32::MAX`. + #[inline] + pub fn write_all(&self, data: &[u8], timeout: Jiffies) -> Result { + if data.len() > i32::MAX as usize { + return Err(EINVAL); + } + + // SAFETY: + // - `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + // - `data.as_ptr()` is guaranteed to be a valid array pointer with the size of + // `data.len()`. + let ret = unsafe { + bindings::serdev_device_write( + self.as_raw(), + data.as_ptr(), + data.len(), + isize::try_from(timeout).unwrap_or_default(), + ) + }; + // CAST: negative return values are guaranteed to be between `-MAX_ERRNO` and `-1`, + // which always fit into a `i32`. + to_result(ret as i32).map(|()| ret.unsigned_abs()) + } + + /// Write data to the serial device. + /// + /// If you want to write until the controller has accepted all the data, use + /// [`Device::write_all`]. + /// + /// Note that any accepted data has only been buffered by the controller. Use + /// [`Device::wait_until_sent`] to make sure the controller write buffer has actually been + /// emptied. + /// + /// Returns the number of bytes written (less than `data.len()` if not enough room in the + /// write buffer). + #[inline] + pub fn write(&self, data: &[u8]) -> Result { + if data.len() > i32::MAX as usize { + return Err(EINVAL); + } + + // SAFETY: + // - `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + // - `data.as_ptr()` is guaranteed to be a valid array pointer with the size of + // `data.len()`. + let ret = + unsafe { bindings::serdev_device_write_buf(self.as_raw(), data.as_ptr(), data.len()) }; + + to_result(ret as i32).map(|()| ret.unsigned_abs()) + } + + /// Send data to the serial device immediately. + /// + /// Note that this doesn't guarantee that the data has been transmitted. + /// Use [`Device::wait_until_sent`] for this purpose. + #[inline] + pub fn write_flush(&self) { + // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + unsafe { bindings::serdev_device_write_flush(self.as_raw()) }; + } + + /// Wait for the data to be sent. + /// + /// After this function, the write buffer of the controller should be empty or the timeout + /// elapsed. + /// + /// Use a timeout of 0 to wait indefinitely. + #[inline] + pub fn wait_until_sent(&self, timeout: Jiffies) { + // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `serdev_device`. + unsafe { + bindings::serdev_device_wait_until_sent( + self.as_raw(), + isize::try_from(timeout).unwrap_or_default(), + ) + }; + } +} + +// SAFETY: `serdev::Device` is a transparent wrapper of `struct serdev_device`. +// The offset is guaranteed to point to a valid device field inside `serdev::Device`. +unsafe impl device::AsBusDevice for Device { + const OFFSET: usize = offset_of!(bindings::serdev_device, dev); +} + +// SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic +// argument. +kernel::impl_device_context_deref!(unsafe { Device }); +kernel::impl_device_context_into_aref!(Device); + +// SAFETY: Instances of `Device` are always reference-counted. +unsafe impl AlwaysRefCounted for Device { + fn inc_ref(&self) { + self.as_ref().inc_ref(); + } + + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::serdev_device_put(obj.cast().as_ptr()) } + } +} + +impl AsRef> for Device { + fn as_ref(&self) -> &device::Device { + // SAFETY: By the type invariant of `Self`, `self.as_raw()` is a pointer to a valid + // `struct serdev_device`. + let dev = unsafe { &raw mut (*self.as_raw()).dev }; + + // SAFETY: `dev` points to a valid `struct device`. + unsafe { device::Device::from_raw(dev) } + } +} + +// SAFETY: A `Device` is always reference-counted and can be released from any thread. +unsafe impl Send for Device {} + +// SAFETY: `Device` can be shared among threads because all methods of `Device` +// (i.e. `Device) are thread safe. +unsafe impl Sync for Device {} + +// SAFETY: Same as `Device` -- the underlying `struct serdev_device` is the same; +// `Bound` is a zero-sized type-state marker that does not affect thread safety. +unsafe impl Sync for Device {} diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index 67b3874cb3d2..132dd428c1f6 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -13,7 +13,10 @@ #[doc(hidden)] pub mod for_lt; -pub use for_lt::ForLt; +pub use for_lt::{ + CovariantForLt, + ForLt, // +}; /// Used to transfer ownership to and from foreign (non-Rust) languages. /// diff --git a/rust/kernel/types/for_lt.rs b/rust/kernel/types/for_lt.rs index d44323c28e8d..b8f422c802dc 100644 --- a/rust/kernel/types/for_lt.rs +++ b/rust/kernel/types/for_lt.rs @@ -1,22 +1,19 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT -//! Provide implementation and test of the `ForLt` trait and macro. +//! Provide implementation and test of the [`trait@ForLt`] and [`trait@CovariantForLt`] traits and +//! macros. //! -//! This module is hidden and user should just use `ForLt!` directly. +//! This module is hidden and users should just use [`ForLt!`](macro@ForLt) / +//! [`CovariantForLt!`](macro@CovariantForLt) directly. use core::marker::PhantomData; /// Representation of types generic over a lifetime. /// -/// The type must be covariant over the generic lifetime, i.e. the lifetime parameter -/// can be soundly shortened. -/// -/// The lifetime involved must be covariant. -/// /// # Macro /// -/// It is not recommended to implement this trait directly. `ForLt!` macro is provided to obtain a -/// type that implements this trait. +/// It is not recommended to implement this trait directly. [`ForLt!`](macro@ForLt) macro is +/// provided to obtain a type that implements this trait. /// /// The full syntax is /// @@ -49,16 +46,65 @@ /// ForLt!(u32) // Equivalent to `ForLt!(for<'a> u32)`. /// # >(); /// ``` +pub trait ForLt { + /// The type parameterized by the lifetime. + type Of<'a>: 'a; +} +pub use macros::ForLt; + +/// [`trait@ForLt`] subtrait for types that are covariant over their lifetime parameter. +/// +/// Provides a safe [`cast_ref`](CovariantForLt::cast_ref) method for types that are proven to be +/// covariant. The `CovariantForLt!` macro syntax is the same as `ForLt!`. +/// +/// # Macro +/// +/// It is not recommended to implement this trait directly. +/// [`CovariantForLt!`](macro@CovariantForLt) macro is provided to obtain a type that implements +/// this trait. +/// +/// The full syntax is +/// +/// ``` +/// # use kernel::types::CovariantForLt; +/// # fn expect_lt() {} +/// # struct TypeThatUse<'a>(&'a ()); +/// # expect_lt::< +/// CovariantForLt!(for<'a> TypeThatUse<'a>) +/// # >(); +/// ``` +/// +/// which gives a type so that +/// ` TypeThatUse<'a>) as CovariantForLt>::Of<'b>` +/// is `TypeThatUse<'b>`. +/// +/// You may also use a short-hand syntax which works similar to lifetime elision. +/// The macro also accepts types that do not involve a lifetime at all. +/// +/// ``` +/// # use kernel::types::CovariantForLt; +/// # fn expect_lt() {} +/// # struct TypeThatUse<'a>(&'a ()); +/// # expect_lt::< +/// CovariantForLt!(TypeThatUse<'_>) // Equivalent to `CovariantForLt!(for<'a> TypeThatUse<'a>)`. +/// # >(); +/// # expect_lt::< +/// CovariantForLt!(&u32) // Equivalent to `CovariantForLt!(for<'a> &'a u32)`. +/// # >(); +/// # expect_lt::< +/// CovariantForLt!(u32) // Equivalent to `CovariantForLt!(for<'a> u32)`. +/// # >(); +/// ``` /// /// The macro will attempt to prove that the type is indeed covariant over the lifetime supplied. /// When it cannot be syntactically proven, it will emit checks to ask the Rust compiler to prove /// it. /// /// ```ignore,compile_fail -/// # use kernel::types::ForLt; -/// # fn expect_lt() {} +/// # use kernel::types::CovariantForLt; +/// # fn expect_lt() {} /// # expect_lt::< -/// ForLt!(fn(&u32)) // Contravariant, will fail compilation. +/// CovariantForLt!(fn(&u32)) // Contravariant, will fail compilation. /// # >(); /// ``` /// @@ -67,26 +113,23 @@ /// the generic parameter but is in a separate item. /// /// ``` -/// # use kernel::types::ForLt; -/// fn expect_lt() {} +/// # use kernel::types::CovariantForLt; +/// fn expect_lt() {} /// # #[allow(clippy::unnecessary_safety_comment, reason = "false positive")] /// fn generic_fn() { /// // Syntactically proven by the macro -/// expect_lt::(); +/// expect_lt::(); /// // Syntactically proven by the macro -/// expect_lt::)>(); +/// expect_lt::)>(); /// // Cannot be syntactically proven, need to check covariance of `KBox` -/// // expect_lt::)>(); +/// // expect_lt::)>(); /// } /// ``` /// /// # Safety /// /// `Self::Of<'a>` must be covariant over the lifetime `'a`. -pub unsafe trait ForLt { - /// The type parameterized by the lifetime. - type Of<'a>: 'a; - +pub unsafe trait CovariantForLt: ForLt { /// Cast a reference to a shorter lifetime. #[inline(always)] fn cast_ref<'r, 'short: 'r, 'long: 'short>(long: &'r Self::Of<'long>) -> &'r Self::Of<'short> { @@ -94,29 +137,33 @@ fn cast_ref<'r, 'short: 'r, 'long: 'short>(long: &'r Self::Of<'long>) -> &'r Sel unsafe { core::mem::transmute(long) } } } -pub use macros::ForLt; +pub use macros::CovariantForLt; /// This is intended to be an "unsafe-to-refer-to" type. /// -/// Must only be used by the `ForLt!` macro. +/// Must only be used by the [`ForLt!`](macro@ForLt) / [`CovariantForLt!`](macro@CovariantForLt) +/// macros. /// /// `T` is the magic `dyn for<'a> WithLt<'a, TypeThatUse<'a>>` generated by macro. /// /// `WF` is a type that the macro can use to assert some specific type is well-formed. /// /// `N` is to provide the macro a place to emit arbitrary items, in case it needs to prove -/// additional properties. +/// additional properties. [`ForLt!`](macro@ForLt) emits `N = 0`; +/// [`CovariantForLt!`](macro@CovariantForLt) emits `N = 1` after a covariance proof. #[doc(hidden)] pub struct UnsafeForLtImpl(PhantomData<(WF, T)>); -// This is a helper trait for implementation `ForLt` to be able to use HRTB. +// This is a helper trait for implementation of `ForLt` / `CovariantForLt` to be able to use HRTB. #[doc(hidden)] pub trait WithLt<'a> { type Of: 'a; } -// SAFETY: In `ForLt!` macro, a covariance proof is generated when naming `UnsafeForLtImpl` -// and it will fail to evaluate if the type is not covariant. -unsafe impl WithLt<'a>, WF> ForLt for UnsafeForLtImpl { +impl WithLt<'a>, WF, const N: usize> ForLt for UnsafeForLtImpl { type Of<'a> = >::Of; } + +// SAFETY: In `CovariantForLt!` macro, a covariance proof is generated in the `N` const generic +// and it will fail to evaluate if the type is not covariant. Only `N = 1` gets this impl. +unsafe impl WithLt<'a>, WF> CovariantForLt for UnsafeForLtImpl {} diff --git a/rust/kernel/usb.rs b/rust/kernel/usb.rs index 870423806e4f..558e15a08478 100644 --- a/rust/kernel/usb.rs +++ b/rust/kernel/usb.rs @@ -89,7 +89,10 @@ extern "C" fn probe_callback( // does not add additional invariants, so it's safe to transmute. let id = unsafe { &*id.cast::() }; - let info = T::ID_TABLE.info(id.index()); + // SAFETY: `id` comes from `T::ID_TABLE` which is of type `IdArray<_, T::IdInfo>`. It + // can also come from dynamic IDs, which will ensure that `driver_data` exists in + // `T::ID_TABLE` or is 0. + let info = unsafe { id.info_unchecked_opt::() }; let data = T::probe(intf, id, info); let dev: &device::Device> = intf.as_ref(); @@ -242,10 +245,6 @@ unsafe impl RawDeviceId for DeviceId { // SAFETY: `DRIVER_DATA_OFFSET` is the offset to the `driver_info` field. unsafe impl RawDeviceIdIndex for DeviceId { const DRIVER_DATA_OFFSET: usize = core::mem::offset_of!(bindings::usb_device_id, driver_info); - - fn index(&self) -> usize { - self.0.driver_info - } } /// [`IdTable`](kernel::device_id::IdTable) type for USB. @@ -254,14 +253,8 @@ fn index(&self) -> usize { /// Create a USB `IdTable` with its alias for modpost. #[macro_export] macro_rules! usb_device_table { - ($table_name:ident, $module_table_name:ident, $id_info_type: ty, $table_data: expr) => { - const $table_name: $crate::device_id::IdArray< - $crate::usb::DeviceId, - $id_info_type, - { $table_data.len() }, - > = $crate::device_id::IdArray::new($table_data); - - $crate::module_device_table!("usb", $module_table_name, $table_name); + ($($tt:tt)*) => { + $crate::module_device_table!("usb", $crate::usb::DeviceId, $($tt)*); }; } @@ -277,7 +270,6 @@ macro_rules! usb_device_table { /// /// kernel::usb_device_table!( /// USB_TABLE, -/// MODULE_USB_TABLE, /// ::IdInfo, /// [ /// (usb::DeviceId::from_id(0x1234, 0x5678), ()), @@ -293,7 +285,7 @@ macro_rules! usb_device_table { /// fn probe<'bound>( /// _interface: &'bound usb::Interface>, /// _id: &usb::DeviceId, -/// _info: &'bound Self::IdInfo, +/// _info: Option<&'bound Self::IdInfo>, /// ) -> impl PinInit, Error> + 'bound { /// Err(ENODEV) /// } @@ -322,7 +314,7 @@ pub trait Driver { fn probe<'bound>( interface: &'bound Interface>, id: &DeviceId, - id_info: &'bound Self::IdInfo, + id_info: Option<&'bound Self::IdInfo>, ) -> impl PinInit, Error> + 'bound; /// USB driver disconnect. diff --git a/rust/macros/for_lt.rs b/rust/macros/for_lt.rs index 364d4113cd10..4372cbad3ec4 100644 --- a/rust/macros/for_lt.rs +++ b/rust/macros/for_lt.rs @@ -154,8 +154,8 @@ fn prove(&mut self, ty: &'a Type) { // Note that if we encounter `&'other_lt T`, then we still need to make sure the type // is wellformed if `T` involves `&'lt`, so we defer to the compiler. // - // This is to block cases like `ForLt!(for<'a> &'static &'a u32)`, as the presence of - // the type implies `'a: 'static` but this is unsound. + // This is to block cases like `CovariantForLt!(for<'a> &'static &'a u32)`, as the + // presence of the type implies `'a: 'static` but this is unsound. Type::Reference(ty) if ty.mutability.is_none() && ty.lifetime.as_ref() == Some(self.0) => { @@ -176,7 +176,12 @@ fn prove(&mut self, ty: &'a Type) { } } -pub(crate) fn for_lt(input: HigherRankedType) -> TokenStream { +/// Shared implementation for both `ForLt!` and `CovariantForLt!`. +/// +/// Both macros run the prover and emit `ProveWf` structs to check well-formedness for all lifetime +/// instances (workaround for ). `CovariantForLt!` +/// additionally emits covariance proof functions and sets `N = 1`. +fn for_lt_inner(input: HigherRankedType, prove_covariance: bool) -> TokenStream { let (ty, lifetime) = match input { HigherRankedType::Explicit { lifetime, ty, .. } => (ty, lifetime), HigherRankedType::Implicit { ty } => { @@ -211,14 +216,16 @@ struct #wf_proof_name<#lifetime>( )); // Insert a proof that the type is covariant. - let cov_proof_name = format_ident!("prove_covariant_{idx}"); - proof.push(quote!( - fn #cov_proof_name<'__short, '__long: '__short>( - long: #wf_proof_name<'__long> - ) -> #wf_proof_name<'__short> { - long - } - )); + if prove_covariance { + let cov_proof_name = format_ident!("prove_covariant_{idx}"); + proof.push(quote!( + fn #cov_proof_name<'__short, '__long: '__short>( + long: #wf_proof_name<'__long> + ) -> #wf_proof_name<'__short> { + long + } + )); + } } // Make sure that the type is wellformed when substituting lifetime with `'static`. @@ -234,6 +241,8 @@ fn #cov_proof_name<'__short, '__long: '__short>( }, ); + let n: usize = prove_covariance.into(); + quote!( ::kernel::types::for_lt::UnsafeForLtImpl::< dyn for<#lifetime> ::kernel::types::for_lt::WithLt<#lifetime, Of = #ty>, @@ -241,8 +250,16 @@ fn #cov_proof_name<'__short, '__long: '__short>( { #(#proof)* - 0 + #n } > ) } + +pub(crate) fn for_lt(input: HigherRankedType) -> TokenStream { + for_lt_inner(input, false) +} + +pub(crate) fn covariant_for_lt(input: HigherRankedType) -> TokenStream { + for_lt_inner(input, true) +} diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs index 408a90567f7e..37a6643db76e 100644 --- a/rust/macros/lib.rs +++ b/rust/macros/lib.rs @@ -515,8 +515,24 @@ pub fn kunit_tests(attr: TokenStream, input: TokenStream) -> TokenStream { /// /// [`ForLt`]: trait.ForLt.html #[proc_macro] -// The macro shares the name with the trait. #[allow(non_snake_case)] pub fn ForLt(input: TokenStream) -> TokenStream { for_lt::for_lt(parse_macro_input!(input)).into() } + +/// Obtain a type that implements [`CovariantForLt`] (and [`ForLt`]) for the given higher-ranked +/// type. +/// +/// Unlike [`ForLt!`], this macro additionally proves that the type is covariant over the lifetime, +/// providing a safe [`CovariantForLt::cast_ref`] method. +/// +/// Please refer to the documentation of the [`CovariantForLt`] trait. +/// +/// [`CovariantForLt`]: trait.CovariantForLt.html +/// [`CovariantForLt::cast_ref`]: trait.CovariantForLt.html#method.cast_ref +/// [`ForLt`]: trait.ForLt.html +#[proc_macro] +#[allow(non_snake_case)] +pub fn CovariantForLt(input: TokenStream) -> TokenStream { + for_lt::covariant_for_lt(parse_macro_input!(input)).into() +} diff --git a/samples/kobject/kobject-example.c b/samples/kobject/kobject-example.c index 36d87ca0bee2..36f3fbc47fe0 100644 --- a/samples/kobject/kobject-example.c +++ b/samples/kobject/kobject-example.c @@ -25,13 +25,13 @@ static int bar; /* * The "foo" file where a static variable is read from and written to. */ -static ssize_t foo_show(struct kobject *kobj, struct kobj_attribute *attr, +static ssize_t foo_show(struct kobject *kobj, const struct kobj_attribute *attr, char *buf) { return sysfs_emit(buf, "%d\n", foo); } -static ssize_t foo_store(struct kobject *kobj, struct kobj_attribute *attr, +static ssize_t foo_store(struct kobject *kobj, const struct kobj_attribute *attr, const char *buf, size_t count) { int ret; @@ -44,14 +44,14 @@ static ssize_t foo_store(struct kobject *kobj, struct kobj_attribute *attr, } /* Sysfs attributes cannot be world-writable. */ -static struct kobj_attribute foo_attribute = - __ATTR(foo, 0664, foo_show, foo_store); +static const struct kobj_attribute foo_attribute = + __KOBJ_ATTR(foo, 0664, foo_show, foo_store); /* * More complex function where we determine which variable is being accessed by * looking at the attribute for the "baz" and "bar" files. */ -static ssize_t b_show(struct kobject *kobj, struct kobj_attribute *attr, +static ssize_t b_show(struct kobject *kobj, const struct kobj_attribute *attr, char *buf) { int var; @@ -63,7 +63,7 @@ static ssize_t b_show(struct kobject *kobj, struct kobj_attribute *attr, return sysfs_emit(buf, "%d\n", var); } -static ssize_t b_store(struct kobject *kobj, struct kobj_attribute *attr, +static ssize_t b_store(struct kobject *kobj, const struct kobj_attribute *attr, const char *buf, size_t count) { int var, ret; @@ -79,17 +79,17 @@ static ssize_t b_store(struct kobject *kobj, struct kobj_attribute *attr, return count; } -static struct kobj_attribute baz_attribute = - __ATTR(baz, 0664, b_show, b_store); -static struct kobj_attribute bar_attribute = - __ATTR(bar, 0664, b_show, b_store); +static const struct kobj_attribute baz_attribute = + __KOBJ_ATTR(baz, 0664, b_show, b_store); +static const struct kobj_attribute bar_attribute = + __KOBJ_ATTR(bar, 0664, b_show, b_store); /* * Create a group of attributes so that we can create and destroy them all * at once. */ -static struct attribute *attrs[] = { +static const struct attribute *const attrs[] = { &foo_attribute.attr, &baz_attribute.attr, &bar_attribute.attr, @@ -103,7 +103,7 @@ static struct attribute *attrs[] = { * attribute group. */ static const struct attribute_group attr_group = { - .attrs = attrs, + .attrs_const = attrs, }; static struct kobject *example_kobj; diff --git a/samples/rust/Kconfig b/samples/rust/Kconfig index c49ab9106345..31d62533ef25 100644 --- a/samples/rust/Kconfig +++ b/samples/rust/Kconfig @@ -161,6 +161,17 @@ config SAMPLE_RUST_DRIVER_AUXILIARY If unsure, say N. +config SAMPLE_RUST_DRIVER_SERDEV + tristate "Serial Device Bus Device Driver" + select RUST_SERIAL_DEV_BUS_ABSTRACTIONS + help + This option builds the Rust serial device bus driver sample. + + To compile this as a module, choose M here: + the module will be called rust_driver_serdev. + + If unsure, say N. + config SAMPLE_RUST_SOC tristate "SoC Driver" select SOC_BUS diff --git a/samples/rust/Makefile b/samples/rust/Makefile index 6c0aaa58cccc..b986b681cde5 100644 --- a/samples/rust/Makefile +++ b/samples/rust/Makefile @@ -14,6 +14,7 @@ obj-$(CONFIG_SAMPLE_RUST_DRIVER_PLATFORM) += rust_driver_platform.o obj-$(CONFIG_SAMPLE_RUST_DRIVER_USB) += rust_driver_usb.o obj-$(CONFIG_SAMPLE_RUST_DRIVER_FAUX) += rust_driver_faux.o obj-$(CONFIG_SAMPLE_RUST_DRIVER_AUXILIARY) += rust_driver_auxiliary.o +obj-$(CONFIG_SAMPLE_RUST_DRIVER_SERDEV) += rust_driver_serdev.o obj-$(CONFIG_SAMPLE_RUST_CONFIGFS) += rust_configfs.o obj-$(CONFIG_SAMPLE_RUST_SOC) += rust_soc.o diff --git a/samples/rust/rust_debugfs.rs b/samples/rust/rust_debugfs.rs index 1f59e08aaa4b..77c19a437695 100644 --- a/samples/rust/rust_debugfs.rs +++ b/samples/rust/rust_debugfs.rs @@ -110,7 +110,6 @@ fn from_str(s: &str) -> Result { kernel::acpi_device_table!( ACPI_TABLE, - MODULE_ACPI_TABLE, ::IdInfo, [(acpi::DeviceId::new(c"LNUXBEEF"), ())] ); @@ -147,7 +146,9 @@ fn build_inner(dir: &Dir) -> impl PinInit>> + '_ { dir.read_write_file(c"pair", new_mutex!(Inner { x: 3, y: 10 })) } - fn new<'a>(pdev: &'a platform::Device>) -> impl PinInit + 'a { + fn new<'a, 'b>( + pdev: &'a platform::Device>, + ) -> impl PinInit + use<'a, 'b> { let debugfs = Dir::new(c"sample_debugfs"); let dev = pdev.as_ref(); diff --git a/samples/rust/rust_debugfs_scoped.rs b/samples/rust/rust_debugfs_scoped.rs index 6a575a15a2c2..ca2b154be384 100644 --- a/samples/rust/rust_debugfs_scoped.rs +++ b/samples/rust/rust_debugfs_scoped.rs @@ -75,7 +75,10 @@ fn create_file_write( GFP_KERNEL, )?; } - let blob = KBox::pin_init(new_mutex!([0x42; SZ_4K]), GFP_KERNEL)?; + let blob = KBox::pin_init( + new_mutex!(pin_init::init_array_from_fn(|_| 0x42)), + GFP_KERNEL, + )?; let scope = KBox::pin_init( mod_data.device_dir.scope( diff --git a/samples/rust/rust_dma.rs b/samples/rust/rust_dma.rs index 5046b4628d0e..bd60034ded23 100644 --- a/samples/rust/rust_dma.rs +++ b/samples/rust/rust_dma.rs @@ -12,10 +12,19 @@ Device, DmaMask, // }, - page, pci, + io::{ + io_project, + io_read, + Io, // + }, + page, + pci, prelude::*, - scatterlist::{Owned, SGTable}, - sync::aref::ARef, + scatterlist::{ + Owned, + SGTable, // + }, + sync::aref::ARef, // }; #[pin_data(PinnedDrop)] @@ -34,6 +43,7 @@ struct DmaSampleDriver { (0xcd, 0xef), ]; +#[derive(FromBytes, IntoBytes)] struct MyStruct { h: u32, b: u32, @@ -51,7 +61,6 @@ unsafe impl kernel::transmute::FromBytes for MyStruct {} kernel::pci_device_table!( PCI_TABLE, - MODULE_PCI_TABLE, ::IdInfo, [(pci::DeviceId::from_id(pci::Vendor::REDHAT, 0x5), ())] ); @@ -63,7 +72,7 @@ impl pci::Driver for DmaSampleDriver { fn probe<'bound>( pdev: &'bound pci::Device>, - _info: &'bound Self::IdInfo, + _info: Option<&'bound Self::IdInfo>, ) -> impl PinInit + 'bound { pin_init::pin_init_scope(move || { dev_info!(pdev, "Probe DMA test driver.\n"); @@ -77,7 +86,7 @@ fn probe<'bound>( Coherent::zeroed_slice(pdev.as_ref(), TEST_VALUES.len(), GFP_KERNEL)?; for (i, value) in TEST_VALUES.into_iter().enumerate() { - kernel::dma_write!(ca, [try: i], MyStruct::new(value.0, value.1)); + io_project!(ca, [panic: i]).copy_write(MyStruct::new(value.0, value.1)); } let size = 4 * page::PAGE_SIZE; @@ -97,8 +106,8 @@ fn probe<'bound>( impl DmaSampleDriver { fn check_dma(&self) { for (i, value) in TEST_VALUES.into_iter().enumerate() { - let val0 = kernel::dma_read!(self.ca, [panic: i].h); - let val1 = kernel::dma_read!(self.ca, [panic: i].b); + let val0 = io_read!(self.ca, [panic: i].h); + let val1 = io_read!(self.ca, [panic: i].b); assert_eq!(val0, value.0); assert_eq!(val1, value.1); diff --git a/samples/rust/rust_driver_auxiliary.rs b/samples/rust/rust_driver_auxiliary.rs index 2c1351040e45..0bee16faecc6 100644 --- a/samples/rust/rust_driver_auxiliary.rs +++ b/samples/rust/rust_driver_auxiliary.rs @@ -11,20 +11,26 @@ Core, // }, driver, + new_mutex, pci, prelude::*, - types::ForLt, + sync::Mutex, + types::{ + CovariantForLt, + ForLt, // + }, InPlaceModule, // }; const MODULE_NAME: &CStr = ::NAME; const AUXILIARY_NAME: &CStr = c"auxiliary"; +const COVARIANT_DEV_ID: u32 = 0; +const INVARIANT_DEV_ID: u32 = 1; struct AuxiliaryDriver; kernel::auxiliary_device_table!( AUX_TABLE, - MODULE_AUX_TABLE, ::IdInfo, [(auxiliary::DeviceId::new(MODULE_NAME, AUXILIARY_NAME), ())] ); @@ -56,17 +62,30 @@ struct Data<'bound> { parent: &'bound pci::Device, } +/// Registration data with interior mutability. +/// +/// `Mutex<&'bound T>` is invariant over `'bound`, so this type cannot implement +/// [`CovariantForLt`](trait@CovariantForLt). Access must go through the closure-based +/// [`auxiliary::Device::registration_data_with()`]. +#[pin_data] +struct MutexData<'bound> { + #[pin] + parent: Mutex<&'bound pci::Device>, + index: u32, +} + struct ParentDriver; #[allow(clippy::type_complexity)] +#[pin_data] struct ParentData<'bound> { - _reg0: auxiliary::Registration<'bound, ForLt!(Data<'_>)>, - _reg1: auxiliary::Registration<'bound, ForLt!(Data<'_>)>, + _reg0: auxiliary::Registration<'bound, CovariantForLt!(Data<'_>)>, + #[pin] + _reg1: auxiliary::Registration<'bound, ForLt!(MutexData<'_>)>, } kernel::pci_device_table!( PCI_TABLE, - MODULE_PCI_TABLE, ::IdInfo, [(pci::DeviceId::from_id(pci::Vendor::REDHAT, 0x5), ())] ); @@ -79,19 +98,19 @@ impl pci::Driver for ParentDriver { fn probe<'bound>( pdev: &'bound pci::Device>, - _info: &'bound Self::IdInfo, + _info: Option<&'bound Self::IdInfo>, ) -> impl PinInit, Error> + 'bound { - Ok(ParentData { + try_pin_init!(ParentData { // SAFETY: `ParentData` is the driver's private data, which is dropped when the // device is unbound; i.e. `mem::forget()` is never called on it. _reg0: unsafe { auxiliary::Registration::new_with_lt( pdev.as_ref(), AUXILIARY_NAME, - 0, + COVARIANT_DEV_ID, MODULE_NAME, Data { - index: 0, + index: COVARIANT_DEV_ID, parent: pdev, }, )? @@ -101,12 +120,16 @@ fn probe<'bound>( auxiliary::Registration::new_with_lt( pdev.as_ref(), AUXILIARY_NAME, - 1, + INVARIANT_DEV_ID, MODULE_NAME, - Data { - index: 1, - parent: pdev, - }, + pin_init!(MutexData { + parent <- { + let pdev: &pci::Device = pdev; + + new_mutex!(pdev) + }, + index: INVARIANT_DEV_ID, + }), )? }, }) @@ -115,22 +138,39 @@ fn probe<'bound>( impl ParentDriver { fn connect(adev: &auxiliary::Device) -> Result { - let data = adev.registration_data::)>()?; - let pdev = data.parent; + match adev.id() { + // CovariantForLt types can use the direct-reference accessor. + COVARIANT_DEV_ID => { + let data = adev.registration_data::)>()?; + let pdev = data.parent; - dev_info!( - pdev, - "Connect auxiliary {} with parent: VendorID={}, DeviceID={:#x}\n", - adev.id(), - pdev.vendor_id(), - pdev.device_id() - ); + dev_info!( + pdev, + "Connect auxiliary {} with parent: VendorID={}, DeviceID={:#x}\n", + adev.id(), + pdev.vendor_id(), + pdev.device_id() + ); - dev_info!( - pdev, - "Connected to auxiliary device with index {}.\n", - data.index - ); + dev_info!( + pdev, + "Connected to auxiliary device with index {}.\n", + data.index + ); + } + // Invariant ForLt types (e.g. containing a Mutex) require the closure-based accessor. + INVARIANT_DEV_ID => { + adev.registration_data_with::), _>(|data| { + let pdev = *data.parent.lock(); + dev_info!( + pdev, + "Connected to auxiliary device with index {} (via Mutex).\n", + data.index + ); + })?; + } + _ => return Err(EINVAL), + } Ok(()) } diff --git a/samples/rust/rust_driver_i2c.rs b/samples/rust/rust_driver_i2c.rs index ead8263a7d48..a0df0c6097c4 100644 --- a/samples/rust/rust_driver_i2c.rs +++ b/samples/rust/rust_driver_i2c.rs @@ -14,21 +14,18 @@ kernel::acpi_device_table! { ACPI_TABLE, - MODULE_ACPI_TABLE, ::IdInfo, [(acpi::DeviceId::new(c"LNUXBEEF"), 0)] } kernel::i2c_device_table! { I2C_TABLE, - MODULE_I2C_TABLE, ::IdInfo, [(i2c::DeviceId::new(c"rust_driver_i2c"), 0)] } kernel::of_device_table! { OF_TABLE, - MODULE_OF_TABLE, ::IdInfo, [(of::DeviceId::new(c"test,rust_driver_i2c"), 0)] } diff --git a/samples/rust/rust_driver_pci.rs b/samples/rust/rust_driver_pci.rs index 1aa8197d8698..2282191e6292 100644 --- a/samples/rust/rust_driver_pci.rs +++ b/samples/rust/rust_driver_pci.rs @@ -74,7 +74,6 @@ struct SampleDriverData<'bound> { kernel::pci_device_table!( PCI_TABLE, - MODULE_PCI_TABLE, ::IdInfo, [( pci::DeviceId::from_id(pci::Vendor::REDHAT, 0x5), @@ -144,7 +143,7 @@ impl pci::Driver for SampleDriver { fn probe<'bound>( pdev: &'bound pci::Device>, - info: &'bound Self::IdInfo, + info: Option<&'bound Self::IdInfo>, ) -> impl PinInit, Error> + 'bound { let vendor = pdev.vendor_id(); dev_dbg!( @@ -153,6 +152,7 @@ fn probe<'bound>( vendor, pdev.device_id() ); + let info = info.ok_or(ENODEV)?; pdev.enable_device_mem()?; pdev.set_master(); diff --git a/samples/rust/rust_driver_platform.rs b/samples/rust/rust_driver_platform.rs index ec0d6cac4f57..710145b3605a 100644 --- a/samples/rust/rust_driver_platform.rs +++ b/samples/rust/rust_driver_platform.rs @@ -87,14 +87,12 @@ struct SampleDriver { kernel::of_device_table!( OF_TABLE, - MODULE_OF_TABLE, ::IdInfo, [(of::DeviceId::new(c"test,rust-device"), Info(42))] ); kernel::acpi_device_table!( ACPI_TABLE, - MODULE_ACPI_TABLE, ::IdInfo, [(acpi::DeviceId::new(c"LNUXBEEF"), Info(0))] ); diff --git a/samples/rust/rust_driver_serdev.rs b/samples/rust/rust_driver_serdev.rs new file mode 100644 index 000000000000..51b4898cd855 --- /dev/null +++ b/samples/rust/rust_driver_serdev.rs @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Rust Serial device bus device driver sample. + +use kernel::{ + acpi, + device::{ + Bound, + Core, // + }, + of, + prelude::*, + serdev, + sync::aref::ARef, // +}; + +struct SampleDriver { + sdev: ARef, +} + +kernel::of_device_table!( + OF_TABLE, + ::IdInfo, + [(of::DeviceId::new(c"test,rust_driver_serdev"), ())] +); + +kernel::acpi_device_table!( + ACPI_TABLE, + ::IdInfo, + [(acpi::DeviceId::new(c"LNUXBEEF"), ())] +); + +#[vtable] +impl serdev::Driver for SampleDriver { + type IdInfo = (); + type Data<'bound> = Self; + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + const ACPI_ID_TABLE: Option> = Some(&ACPI_TABLE); + + fn probe<'bound>( + sdev: &'bound serdev::Device>, + _info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit + 'bound { + let dev = sdev.as_ref(); + + dev_dbg!(dev, "Probe Rust Serial device bus device driver sample.\n"); + + if sdev + .set_baudrate( + dev.fwnode() + .and_then(|fwnode| fwnode.property_read(c"baudrate").optional()) + .unwrap_or(115200), + ) + .is_err() + { + return Err(EINVAL); + } + sdev.set_flow_control(false); + sdev.set_parity(serdev::Parity::None)?; + + Ok(Self { sdev: sdev.into() }) + } + + fn receive<'bound>( + sdev: &'bound serdev::Device, + _this: Pin<&Self>, + data: &[u8], + ) -> usize { + sdev.write(data).unwrap_or_default() as usize + } +} + +impl Drop for SampleDriver { + fn drop(&mut self) { + dev_dbg!( + self.sdev.as_ref(), + "Remove Rust Serial device bus device driver sample.\n" + ); + } +} + +kernel::module_serdev_device_driver! { + type: SampleDriver, + name: "rust_driver_serdev", + authors: ["Markus Probst"], + description: "Rust Serial device bus device driver", + license: "GPL v2", +} diff --git a/samples/rust/rust_driver_usb.rs b/samples/rust/rust_driver_usb.rs index 02bd5085f9bc..7ef04e177c80 100644 --- a/samples/rust/rust_driver_usb.rs +++ b/samples/rust/rust_driver_usb.rs @@ -19,7 +19,6 @@ struct SampleDriver { kernel::usb_device_table!( USB_TABLE, - MODULE_USB_TABLE, ::IdInfo, [(usb::DeviceId::from_id(0x1234, 0x5678), ()),] ); @@ -32,7 +31,7 @@ impl usb::Driver for SampleDriver { fn probe<'bound>( intf: &'bound usb::Interface>, _id: &usb::DeviceId, - _info: &'bound Self::IdInfo, + _info: Option<&'bound Self::IdInfo>, ) -> impl PinInit + 'bound { let dev: &device::Device> = intf.as_ref(); dev_info!(dev, "Rust USB driver sample probed\n"); diff --git a/samples/rust/rust_i2c_client.rs b/samples/rust/rust_i2c_client.rs index 2d876f4e3ee0..c8a23875ef5b 100644 --- a/samples/rust/rust_i2c_client.rs +++ b/samples/rust/rust_i2c_client.rs @@ -87,14 +87,12 @@ struct SampleDriver { kernel::of_device_table!( OF_TABLE, - MODULE_OF_TABLE, ::IdInfo, [(of::DeviceId::new(c"test,rust-device"), ())] ); kernel::acpi_device_table!( ACPI_TABLE, - MODULE_ACPI_TABLE, ::IdInfo, [(acpi::DeviceId::new(c"LNUXBEEF"), ())] ); diff --git a/samples/rust/rust_soc.rs b/samples/rust/rust_soc.rs index 808d58200eb6..f5e5f2f9adf7 100644 --- a/samples/rust/rust_soc.rs +++ b/samples/rust/rust_soc.rs @@ -23,14 +23,12 @@ struct SampleSocDriver { kernel::of_device_table!( OF_TABLE, - MODULE_OF_TABLE, ::IdInfo, [(of::DeviceId::new(c"test,rust-device"), ())] ); kernel::acpi_device_table!( ACPI_TABLE, - MODULE_ACPI_TABLE, ::IdInfo, [(acpi::DeviceId::new(c"LNUXBEEF"), ())] );