diff --git a/.mailmap b/.mailmap index 1f5540bc33f1..b8d0dc185714 100644 --- a/.mailmap +++ b/.mailmap @@ -286,6 +286,8 @@ Ed L. Cashin Elliot Berman Enric Balletbo i Serra Enric Balletbo i Serra +Eric Dumazet +Eric Dumazet Erik Kaneda Ethan Carter Edwards Ethan Edwards Eugen Hristev @@ -355,6 +357,8 @@ Hans Verkuil Hans Verkuil Hans Verkuil Hao Ge +Haowen Bai +Haowen Bai Harry Yoo <42.hyeyoo@gmail.com> Harry Yoo Heiko Carstens @@ -975,6 +979,7 @@ Vladimir Davydov Vlastimil Babka WangYuli WangYuli +Wei Wang Weiwen Hu WeiXiong Liao Wen Gong @@ -982,6 +987,8 @@ Wesley Cheng Will Deacon Wolfram Sang Wolfram Sang +Xu Xin +Xu Xin xu xin Yakir Yang Yanteng Si Ying Huang diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 33cd30996e47..e75344f4e0cd 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -589,6 +589,9 @@ Kernel parameters arm64.nosve [ARM64] Unconditionally disable Scalable Vector Extension support + arm64.nowfxt [ARM64] Unconditionally disable Wait For Event with + Timeout and Wait For Interrupt with Timeout support + ataflop= [HW,M68k] atarimouse= [HW,MOUSE] Atari Mouse @@ -3491,7 +3494,7 @@ Kernel parameters * SATA link speed limit: 1.5Gbps or 3.0Gbps. - * Transfer mode: pio[0-7], mwdma[0-4] and udma[0-7]. + * Transfer mode: pio[0-6], mwdma[0-4] and udma[0-7]. udma[/][16,25,33,44,66,100,133] notation is also allowed. @@ -3509,7 +3512,7 @@ Kernel parameters * [no]ncqtrim: Enable or disable queued DSM TRIM. - * [no]ncqati: Enable or disable NCQ trim on ATI chipset. + * [no]ncqati: Enable or disable NCQ on ATI chipsets. * [no]trim: Enable or disable (unqueued) TRIM. @@ -3540,11 +3543,9 @@ Kernel parameters * max_sec_128: Set transfer size limit to 128 sectors. - * max_sec_1024: Set or clear transfer size limit to - 1024 sectors. + * max_sec_1024: Set transfer size limit to 1024 sectors. - * max_sec_lba48: Set or clear transfer size limit to - 65535 sectors. + * max_sec_lba48: Set transfer size limit to 65535 sectors. * external: Mark port as external (hotplug-capable). diff --git a/Documentation/arch/arm64/booting.rst b/Documentation/arch/arm64/booting.rst index 13ef311dace8..3fea4b14ef7c 100644 --- a/Documentation/arch/arm64/booting.rst +++ b/Documentation/arch/arm64/booting.rst @@ -465,6 +465,7 @@ Before jumping into the kernel, the following conditions must be met: - HDFGWTR2_EL2.nPMICNTR_EL0 (bit 2) must be initialised to 0b1. - HDFGWTR2_EL2.nPMICFILTR_EL0 (bit 3) must be initialised to 0b1. - HDFGWTR2_EL2.nPMUACR_EL1 (bit 4) must be initialised to 0b1. + - HDFGWTR2_EL2.nPMZR_EL0 (bit 21) must be initialised to 0b1. For CPUs with SPE data source filtering (FEAT_SPE_FDS): diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index ac3248b9f2f3..99a1eb4b833d 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst @@ -174,6 +174,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A725 | #3456106 | ARM64_ERRATUM_3194386 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A725 | #3821522 | ARM64_ERRATUM_3821522 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X1 | #1502854 | N/A | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-X1 | #3324344 | ARM64_ERRATUM_3194386 | diff --git a/Documentation/arch/s390/pci.rst b/Documentation/arch/s390/pci.rst index 80f4ba193159..565434626fb5 100644 --- a/Documentation/arch/s390/pci.rst +++ b/Documentation/arch/s390/pci.rst @@ -67,7 +67,7 @@ Entries specific to zPCI functions and entries that hold zPCI information. A physical function that currently supports a virtual function cannot be powered off until all virtual functions are removed with: - echo 0 > /sys/bus/pci/devices/DDDD:BB:dd.f/sriov_numvf + echo 0 > /sys/bus/pci/devices/DDDD:BB:dd.f/sriov_numvfs * /sys/bus/pci/devices/DDDD:BB:dd.f/: diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst index 85f73e0bbd0f..6691fe8a32c3 100644 --- a/Documentation/bpf/kfuncs.rst +++ b/Documentation/bpf/kfuncs.rst @@ -486,6 +486,16 @@ Example usage in BPF program: /* note that the last argument is omitted */ bpf_task_work_schedule_signal(task, &work->tw, &arrmap, task_work_callback); +2.5.10 KF_PERFMON flag +---------------------- + +The KF_PERFMON flag is used for kfuncs that can expose kernel memory or kernel +addresses to the BPF program, for example by reading through a pointer that the +verifier does not check. Calling such a kfunc requires CAP_PERFMON, or +CAP_SYS_ADMIN, in the same way that the equivalent BPF helpers are gated in +bpf_base_func_proto(). A program loaded with CAP_BPF alone is rejected at load +time. + 2.6 Registering the kfuncs -------------------------- diff --git a/Documentation/devicetree/bindings/iio/adc/rockchip-saradc.yaml b/Documentation/devicetree/bindings/iio/adc/rockchip-saradc.yaml index f0551d665916..0cce357cb01d 100644 --- a/Documentation/devicetree/bindings/iio/adc/rockchip-saradc.yaml +++ b/Documentation/devicetree/bindings/iio/adc/rockchip-saradc.yaml @@ -12,19 +12,19 @@ maintainers: properties: compatible: oneOf: - - const: rockchip,saradc - - const: rockchip,rk3066-tsadc - - const: rockchip,rk3399-saradc - - const: rockchip,rk3528-saradc + - enum: + - rockchip,saradc + - rockchip,rk3066-tsadc + - rockchip,rk3399-saradc + - rockchip,rk3528-saradc + - rockchip,rk3562-saradc + - rockchip,rk3588-saradc + - rockchip,rv1106-saradc - items: - const: rockchip,rk3506-saradc - const: rockchip,rk3528-saradc - - const: rockchip,rk3562-saradc - - const: rockchip,rk3588-saradc - items: - - enum: - - rockchip,rk3576-saradc - - rockchip,rv1106-saradc + - const: rockchip,rk3576-saradc - const: rockchip,rk3588-saradc - items: - enum: diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,nord-tlmm.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,nord-tlmm.yaml index 4bb511719f31..56758a85ead8 100644 --- a/Documentation/devicetree/bindings/pinctrl/qcom,nord-tlmm.yaml +++ b/Documentation/devicetree/bindings/pinctrl/qcom,nord-tlmm.yaml @@ -67,7 +67,7 @@ $defs: Specify the alternative function to be configured for the specified pins. - enum: [ aoss_cti, atest_char, atest_usb20, atest_usb21, + enum: [ gpio, aoss_cti, atest_char, atest_usb20, atest_usb21, aud_intfc0_clk, aud_intfc0_data, aud_intfc0_ws, aud_intfc10_clk, aud_intfc10_data, aud_intfc10_ws, aud_intfc1_clk, aud_intfc1_data, aud_intfc1_ws, @@ -98,9 +98,10 @@ $defs: pcie3_clk_req_n, phase_flag, pll_bist_sync, pll_clk_aux, prng_rosc0, prng_rosc1, pwrbrk_i_n, qdss, qdss_cti, qspi, qup0_se0, qup0_se1, qup0_se2, qup0_se3, qup0_se4, qup0_se5, - qup1_se0, qup1_se1, qup1_se3, qup1_se2, qup1_se4, qup1_se5, + qup1_se0, qup1_se1, qup1_se2_01, qup1_se2_23, qup1_se3_01, + qup1_se3_23, qup1_se4, qup1_se5, qup1_se6, qup2_se0, qup2_se1, qup2_se2, qup2_se3, qup2_se4, - qup2_se5, qup2_se6, + qup2_se5, qup2_se6, qup3_se0_mira, qup3_se0_mirb, sailss_ospi, sdc4_clk, sdc4_cmd, sdc4_data, smb_alert, smb_alert_n, smb_clk, smb_dat, tb_trig_sdc4, tmess_prng0, tmess_prng1, tsc_timer, tsense_pwm, usb0_hs, diff --git a/Documentation/filesystems/netfs_library.rst b/Documentation/filesystems/netfs_library.rst index ddd799df6ce3..a9de281db8ee 100644 --- a/Documentation/filesystems/netfs_library.rst +++ b/Documentation/filesystems/netfs_library.rst @@ -451,6 +451,32 @@ one. The inode should be marked ``NETFS_ICTX_SINGLE_NO_UPLOAD`` if this API is to be used. The writeback function requires the buffer to be of ITER_FOLIOQ type. +Clearing Stale Post-EOF Pagecache +--------------------------------- + +When a file is extended, data left in the pagecache past the old EOF by a write +through an mmap must not be exposed as file content. Netfslib clears this on +its own write paths, and exports a helper so a filesystem can do the same from +a resize path (truncate, setattr, fallocate and the like) that holds the +inode's ``i_rwsem`` exclusively across the whole resize:: + + void netfs_clear_stale_post_isize(struct inode *inode, uoff_t from, + uoff_t to); + +This zeroes any such data within the ``[from, to)`` hole to be made, where +@from is the old EOF and @to is the new one, and the caller must have already +updated ``i_size`` to @to before calling it. Only the folio straddling @from +can hold data written past the EOF through an mmap, as pages wholly beyond the +EOF can't be faulted in, so the zeroing is limited to that folio. The folio is +zeroed rather than dropped so that a concurrent extending write can't lose +data. + +This overlaps with ``pagecache_isize_extended()`` but can't reuse it: that +helper is keyed on a sub-page block size and is a no-op when the block size is +``>= PAGE_SIZE`` (as on network filesystems), and it doesn't wait for +writeback. As both address the same problem, a change to one should probably +be reflected in the other to keep them in sync. + High-Level VM API ================== diff --git a/Documentation/netlink/specs/netdev.yaml b/Documentation/netlink/specs/netdev.yaml index 3e3f03bd5c29..0562ba1f567a 100644 --- a/Documentation/netlink/specs/netdev.yaml +++ b/Documentation/netlink/specs/netdev.yaml @@ -851,6 +851,8 @@ operations: name: bind-tx doc: Bind dmabuf to netdev for TX attribute-set: dmabuf + # Intentionally unprivileged (no admin-perm / uns-admin-perm); see + # comment above netdev_nl_bind_tx_doit(). do: request: attributes: diff --git a/Documentation/trace/events-landlock.rst b/Documentation/trace/events-landlock.rst index af9267cca47d..304a8d06273e 100644 --- a/Documentation/trace/events-landlock.rst +++ b/Documentation/trace/events-landlock.rst @@ -6,7 +6,7 @@ Landlock Trace Events ===================== :Author: Mickaël Salaün -:Date: August 2026 +:Date: September 2026 Landlock emits trace events for sandbox lifecycle operations and access denials. These events can be consumed by ftrace (for human-readable @@ -31,8 +31,8 @@ Landlock trace events are organized in four categories: **Syscall events** are emitted during Landlock system calls: - ``landlock_create_ruleset``: a new ruleset is created -- ``landlock_add_rule_fs``: a filesystem rule is added to a ruleset -- ``landlock_add_rule_net``: a network port rule is added to a ruleset +- ``landlock_add_rule_path_beneath``: a filesystem rule is added to a ruleset +- ``landlock_add_rule_net_port``: a network port rule is added to a ruleset - ``landlock_create_domain``: a new domain is created from a ruleset - ``landlock_enforce_domain``: a domain is enforced on a thread @@ -47,8 +47,8 @@ Landlock trace events are organized in four categories: **Rule evaluation events** are emitted during rule matching: -- ``landlock_check_rule_fs``: a filesystem rule is evaluated -- ``landlock_check_rule_net``: a network port rule is evaluated +- ``landlock_check_rule_inode``: an inode-keyed rule is evaluated +- ``landlock_check_rule_net_port``: a network-port-keyed rule is evaluated **Lifecycle events**: @@ -141,10 +141,10 @@ in some field formats: treated as untrusted input and escaped in the trace text output so it cannot inject field separators or control characters. -- **Other party's domain**: A scope or ptrace denial compares the - subject's denying domain (``domain=``, always the enforcing domain and - never the current task) with the other party's domain, so these - tracepoints also report the other party's domain as a scalar ID: +- **Other party's domain**: A scope or ptrace denial compares the subject's + denying domain (``domain=``), which is the enforcing domain and not + necessarily the current task's domain, with the other party's domain. + These tracepoints also report the other party's domain as a scalar ID: ``tracee_domain=`` (ptrace), ``target_domain=`` (signal), and ``peer_domain=`` (abstract unix socket). It is ``0`` when the other party is unsandboxed, and otherwise a domain ID that a consumer resolves @@ -189,7 +189,7 @@ rather than the caller's, so correlate those to the syscall by domain ID. Interpreting check_rule events ============================== -The ``check_rule_fs`` and ``check_rule_net`` events expose the per-layer +The ``check_rule_inode`` and ``check_rule_net_port`` events expose the per-layer rule evaluation, which is useful for understanding *why* a specific access is allowed or denied. @@ -231,9 +231,10 @@ contribution, not the final decision: through another matching rule, or the right is denied and appears in the ``blockers=`` field of the corresponding ``deny_access`` event. -To reconstruct the decision for an object, aggregate the ``grants=`` -groups of all ``check_rule`` events emitted for that object during the -check. +For an access check that a consumer can delimit, aggregate the ``grants=`` +groups of all matching ``check_rule`` events. These events do not carry a +request ID; use their execution context and generic tracepoints to separate +concurrent or successive checks of the same object. .. note:: @@ -244,29 +245,31 @@ check. For example, a program sandboxed with read and execute access to the whole filesystem reads ``/etc/passwd``; both the ``execve()`` and the -read match the rule covering ``/`` (inode 2), so ``check_rule_fs`` fires +read match the rule covering ``/`` (inode 2), so ``check_rule_inode`` fires with the requested rights intersected against what that rule grants. The ``access_request=`` mask includes ``truncate`` because the file-open hook evaluates that optional right alongside the required access, but the rule does not grant it, so ``truncate`` never appears in ``grants=``:: - cat-127 [...] landlock_check_rule_fs: domain=1e40cb56f access_request=execute|read_file|truncate dev=0:17 ino=2 grants={execute|read_file} - cat-127 [...] landlock_check_rule_fs: domain=1e40cb56f access_request=read_file|truncate dev=0:17 ino=2 grants={read_file} + cat-127 [...] landlock_check_rule_inode: domain=1e40cb56f access_request=execute|read_file|truncate dev=0:17 ino=2 grants={execute|read_file} + cat-127 [...] landlock_check_rule_inode: domain=1e40cb56f access_request=read_file|truncate dev=0:17 ino=2 grants={read_file} The ``[...]`` replaces the ftrace CPU, flags, and timestamp columns. A single ``grants=`` group means the enforcing domain has one layer. With two nested sandboxes that each grant the same rights, the rule spans both layers, so ``grants=`` has one group per layer:: - cat-128 [...] landlock_check_rule_fs: domain=184788b52 access_request=execute|read_file|truncate dev=0:17 ino=2 grants={execute|read_file,execute|read_file} - cat-128 [...] landlock_check_rule_fs: domain=184788b52 access_request=read_file|truncate dev=0:17 ino=2 grants={read_file,read_file} + cat-128 [...] landlock_check_rule_inode: domain=184788b52 access_request=execute|read_file|truncate dev=0:17 ino=2 grants={execute|read_file,execute|read_file} + cat-128 [...] landlock_check_rule_inode: domain=184788b52 access_request=read_file|truncate dev=0:17 ino=2 grants={read_file,read_file} eBPF access =========== -eBPF programs attached via ``BPF_RAW_TRACEPOINT`` can access the -tracepoint arguments directly through BTF. The arguments include both -standard kernel objects and Landlock-internal objects: +BTF-enabled raw tracepoint programs attached through libbpf +``SEC("tp_btf/...")`` sections receive typed callback arguments. The +event prototypes in `Event reference`_ document their argument layouts. +The arguments include both standard kernel objects and Landlock-internal +objects: - Standard kernel objects (``struct task_struct``, ``struct sock``, ``struct path``, ``struct dentry``) can be used with existing BPF @@ -277,7 +280,7 @@ standard kernel objects and Landlock-internal objects: Internal struct layouts may change between kernel versions; use CO-RE for field relocation. -A stateful eBPF program observes the full event stream and maintains +A stateful eBPF program attached before sandbox construction can maintain per-domain state in BPF maps: 1. On ``landlock_create_domain``: record the domain ID and parent (the @@ -293,7 +296,10 @@ per-domain state in BPF maps: final statistics. This approach requires no kernel modification and no Landlock-specific -BPF helpers. The Landlock IDs serve as correlation keys across events. +BPF helpers. Landlock IDs serve as correlation keys within one boot. +Records exported through tracing or BPF buffers can be lost, and records +from different CPUs are not globally ordered, so consumers must detect and +reconcile incomplete state. Audit filtering equivalence =========================== diff --git a/MAINTAINERS b/MAINTAINERS index 4d108339f44e..5e20f9d0dddb 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5143,6 +5143,7 @@ L: netdev@vger.kernel.org L: openwrt-devel@lists.openwrt.org (subscribers-only) S: Supported F: Documentation/devicetree/bindings/net/dsa/brcm,b53.yaml +F: Documentation/networking/dsa/b53.rst F: drivers/net/dsa/b53/* F: drivers/net/dsa/bcm_sf2* F: include/linux/dsa/brcm.h @@ -5454,6 +5455,7 @@ F: include/linux/brcmphy.h BROADCOM GENET ETHERNET DRIVER M: Doug Berger M: Florian Fainelli +M: Nicolai Buchwitz R: Broadcom internal kernel review list L: netdev@vger.kernel.org S: Maintained @@ -14944,7 +14946,7 @@ M: Damien Le Moal M: Niklas Cassel L: linux-ide@vger.kernel.org S: Maintained -T: git git://git.kernel.org/pub/scm/linux/kernel/git/libata/linux.git +T: git git://git.kernel.org/pub/scm/linux/kernel/git/libata/linux.git for-next F: Documentation/ABI/testing/sysfs-ata F: Documentation/devicetree/bindings/ata/ F: drivers/ata/ @@ -17233,7 +17235,7 @@ F: tools/testing/selftests/mm/gup_test.c MEMORY MANAGEMENT - KSM (Kernel Samepage Merging) M: Andrew Morton M: David Hildenbrand -R: Xu Xin +R: Xu Xin R: Chengming Zhou L: linux-mm@kvack.org S: Maintained @@ -17247,31 +17249,6 @@ F: include/trace/events/ksm.h F: mm/ksm.c F: mm/mm_slot.h -MEMORY MANAGEMENT - MEMORY POLICY AND MIGRATION -M: Andrew Morton -M: David Hildenbrand -R: Zi Yan -R: Matthew Brost -R: Joshua Hahn -R: Rakie Kim -R: Byungchul Park -R: Gregory Price -R: Ying Huang -R: Alistair Popple -L: linux-mm@kvack.org -S: Maintained -W: http://www.linux-mm.org -T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm -F: Documentation/ABI/testing/sysfs-kernel-mm-mempolicy -F: Documentation/ABI/testing/sysfs-kernel-mm-mempolicy-weighted-interleave -F: include/linux/mempolicy.h -F: include/uapi/linux/mempolicy.h -F: include/linux/migrate.h -F: include/linux/migrate_mode.h -F: mm/mempolicy.c -F: mm/migrate.c -F: mm/migrate_device.c - MEMORY MANAGEMENT - MGLRU (MULTI-GEN LRU) M: Andrew Morton R: Kairui Song @@ -17281,6 +17258,8 @@ R: Barry Song R: Axel Rasmussen R: Yuanchu Xie R: Wei Xu +R: Baoquan He +R: Baolin Wang L: linux-mm@kvack.org S: Maintained W: http://www.linux-mm.org @@ -17292,6 +17271,25 @@ F: include/linux/mmzone.h F: mm/vmscan.c F: mm/workingset.c +MEMORY MANAGEMENT - MIGRATION +M: Andrew Morton +M: David Hildenbrand +R: Zi Yan +R: Matthew Brost +R: Joshua Hahn +R: Byungchul Park +R: Gregory Price +R: Ying Huang +R: Alistair Popple +L: linux-mm@kvack.org +S: Maintained +W: http://www.linux-mm.org +T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm +F: include/linux/migrate.h +F: include/linux/migrate_mode.h +F: mm/migrate.c +F: mm/migrate_device.c + MEMORY MANAGEMENT - MISC M: Andrew Morton M: David Hildenbrand @@ -17307,7 +17305,6 @@ W: http://www.linux-mm.org T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm F: Documentation/ABI/testing/sysfs-kernel-mm F: Documentation/ABI/testing/sysfs-kernel-mm-cma -F: Documentation/ABI/testing/sysfs-kernel-mm-memory-tiers F: Documentation/ABI/testing/sysfs-kernel-mm-numa F: Documentation/admin-guide/mm/ F: Documentation/mm/ @@ -17315,7 +17312,6 @@ F: drivers/char/mem.c F: include/linux/cma.h F: include/linux/dmapool.h F: include/linux/ioremap.h -F: include/linux/memory-tiers.h F: include/linux/page_idle.h F: mm/backing-dev.c F: mm/cma.c @@ -17327,7 +17323,6 @@ F: mm/early_ioremap.c F: mm/fadvise.c F: mm/ioremap.c F: mm/mapping_dirty_helpers.c -F: mm/memory-tiers.c F: mm/page_idle.c F: mm/pgalloc-track.h F: mm/process_vm_access.c @@ -17344,6 +17339,27 @@ F: mm/numa.c F: mm/numa_emulation.c F: mm/numa_memblks.c +MEMORY MANAGEMENT - NUMA PLACEMENT +M: Andrew Morton +M: David Hildenbrand +M: Gregory Price +R: Zi Yan +R: Joshua Hahn +R: Rakie Kim +R: Ying Huang +L: linux-mm@kvack.org +S: Maintained +W: http://www.linux-mm.org +T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm +F: Documentation/ABI/testing/sysfs-kernel-mm-memory-tiers +F: Documentation/ABI/testing/sysfs-kernel-mm-mempolicy +F: Documentation/ABI/testing/sysfs-kernel-mm-mempolicy-weighted-interleave +F: include/linux/memory-tiers.h +F: include/linux/mempolicy.h +F: include/uapi/linux/mempolicy.h +F: mm/memory-tiers.c +F: mm/mempolicy.c + MEMORY MANAGEMENT - OOM KILLER M: Michal Hocko R: David Rientjes @@ -18921,7 +18937,7 @@ F: net/core/drop_monitor.c NETWORKING DRIVERS M: Andrew Lunn M: "David S. Miller" -M: Eric Dumazet +M: Eric Dumazet M: Jakub Kicinski M: Paolo Abeni L: netdev@vger.kernel.org @@ -19038,7 +19054,7 @@ K: struct\s+phy_port|phy_port_ NETWORKING [GENERAL] M: "David S. Miller" -M: Eric Dumazet +M: Eric Dumazet M: Jakub Kicinski M: Paolo Abeni R: Simon Horman @@ -19264,7 +19280,7 @@ F: net/ipv6/seg6* F: tools/testing/selftests/net/srv6* NETWORKING [TCP] -M: Eric Dumazet +M: Eric Dumazet M: Neal Cardwell R: Kuniyuki Iwashima L: netdev@vger.kernel.org @@ -19297,7 +19313,7 @@ F: net/tls/ F: tools/testing/selftests/net/tls.c NETWORKING [SOCKETS] -M: Eric Dumazet +M: Eric Dumazet M: Kuniyuki Iwashima M: Paolo Abeni M: Willem de Bruijn @@ -20593,6 +20609,7 @@ ORACLE CLUSTER FILESYSTEM 2 (OCFS2) M: Mark Fasheh M: Joel Becker M: Joseph Qi +R: Heming Zhao L: ocfs2-devel@lists.linux.dev S: Supported W: http://ocfs2.wiki.kernel.org @@ -24511,6 +24528,11 @@ R: Marc Murphy S: Supported F: arch/arm/boot/dts/ti/omap/am335x-sancloud* +SB1250-DUART BROADCOM SOC SERIAL DRIVER +M: "Maciej W. Rozycki" +S: Maintained +F: drivers/tty/serial/sb1250-duart.c + SBOM M: Luis Augenstein M: Maximilian Huber @@ -25221,7 +25243,7 @@ R: David Rientjes R: Roman Gushchin L: linux-mm@kvack.org S: Maintained -T: git git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab.git +T: git git://git.kernel.org/pub/scm/linux/kernel/git/mm/slab.git F: Documentation/admin-guide/mm/slab.rst F: Documentation/mm/slab.rst F: include/linux/mempool.h diff --git a/Makefile b/Makefile index 751a08643bf8..7c855e6fe544 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ VERSION = 7 PATCHLEVEL = 3 SUBLEVEL = 0 -EXTRAVERSION = -rc4 +EXTRAVERSION = -rc6 NAME = Baby Opossum Posse # *DOCUMENTATION* diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig index 2ed7186c81c5..80e61175bf52 100644 --- a/arch/arc/Kconfig +++ b/arch/arc/Kconfig @@ -331,13 +331,9 @@ config ARC_FPU_SAVE_RESTORE for floating point exceptions and rounding modes. These are preserved across task context switch when enabled. -config ARC_CANT_LLSC - def_bool n - config ARC_HAS_LLSC bool "Insn: LLOCK/SCOND (efficient atomic ops)" default y - depends on !ARC_CANT_LLSC if ISA_ARCV2 diff --git a/arch/arc/include/asm/atomic64-arcv2.h b/arch/arc/include/asm/atomic64-arcv2.h index 73080a664369..1ca4a1047fb3 100644 --- a/arch/arc/include/asm/atomic64-arcv2.h +++ b/arch/arc/include/asm/atomic64-arcv2.h @@ -156,6 +156,12 @@ static inline u64 __arch_cmpxchg64_relaxed(volatile void *ptr, u64 old, u64 new) } #define arch_cmpxchg64_relaxed __arch_cmpxchg64_relaxed +static inline s64 arch_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) +{ + return arch_cmpxchg64_relaxed(&v->counter, old, new); +} +#define arch_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg_relaxed + static inline s64 arch_atomic64_xchg(atomic64_t *ptr, s64 new) { s64 prev; diff --git a/arch/arc/include/asm/cmpxchg.h b/arch/arc/include/asm/cmpxchg.h index 76f43db0890f..66b3743c9400 100644 --- a/arch/arc/include/asm/cmpxchg.h +++ b/arch/arc/include/asm/cmpxchg.h @@ -46,7 +46,7 @@ __typeof__(*(ptr)) _n_ = (new); \ __typeof__(*(ptr)) _prev_; \ \ - switch(sizeof((_p_))) { \ + switch (sizeof(*(_p_))) { \ case 1: \ _prev_ = (__typeof__(*(ptr)))cmpxchg_emu_u8((volatile u8 *__force)_p_, (uintptr_t)_o_, (uintptr_t)_n_); \ break; \ @@ -102,7 +102,7 @@ __typeof__(ptr) _p_ = (ptr); \ __typeof__(*(ptr)) _val_ = (val); \ \ - switch(sizeof(*(_p_))) { \ + switch (sizeof(*(_p_))) { \ case 4: \ _val_ = __arch_xchg(_p_, _val_); \ break; \ diff --git a/arch/arc/kernel/setup.c b/arch/arc/kernel/setup.c index 7b6a9beba9db..8bebd0e6fc2f 100644 --- a/arch/arc/kernel/setup.c +++ b/arch/arc/kernel/setup.c @@ -593,6 +593,7 @@ static int show_cpuinfo(struct seq_file *m, void *v) cpu_id); } else { freq = clk_get_rate(cpu_clk); + clk_put(cpu_clk); } if (freq) seq_printf(m, "CPU speed\t: %lu.%02lu Mhz\n", diff --git a/arch/arc/kernel/smp.c b/arch/arc/kernel/smp.c index 2d99dffed0ce..ee210a94ce45 100644 --- a/arch/arc/kernel/smp.c +++ b/arch/arc/kernel/smp.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b5a51b0ef944..cab741a695f5 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1040,9 +1040,17 @@ config ARM64_ERRATUM_1902691 If unsure, say Y. +config ARM64_WORKAROUND_BROKEN_AMU_CONSTCNT + bool + # Targeting errata affecting AMEVCNTR01, the AMU constant counter. + # On affected CPUs the counter may not increment at the expected rate, + # making it unreliable. As a result, AMU users should treat it as + # unavailable rather than relying on misleading counter values. + config ARM64_ERRATUM_2457168 bool "Cortex-A510: 2457168: workaround for AMEVCNTR01 incrementing incorrectly" depends on ARM64_AMU_EXTN + select ARM64_WORKAROUND_BROKEN_AMU_CONSTCNT default y help This option adds the workaround for ARM Cortex-A510 erratum 2457168. @@ -1073,6 +1081,28 @@ config ARM64_ERRATUM_2645198 If unsure, say Y. +config ARM64_ERRATUM_3821522 + bool "Cortex-A725: 3821522: workaround for possible CNT_CYCLES increment error due to WFE/WFI" + depends on ARM64_AMU_EXTN + select ARM64_WORKAROUND_BROKEN_AMU_CONSTCNT + default y + help + This option adds the workaround for ARM Cortex-A725 erratum 3821522. + + On affected A725 cores, the CNT_CYCLES event may incur a significant + increment error when entering and subsequently exiting WFx. + As a result, the CNT_CYCLES may diverge from the system counter + frequency at which it is expected to increment. + This renders the AMU counter AMEVCNTR01, that implements CNT_CYCLES, + being unreliable and unsuitable for use. + + Since there is no hardware workaround, reads of the affected CNT_CYCLES + counter return 0 in the relevant paths. This causes users of the counter + to treat it as unavailable and is functionally equivalent to firmware + disabling the affected counter. + + If unsure, say Y. + config ARM64_WORKAROUND_SPECULATIVE_UNPRIV_LOAD bool diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h index aa8ec9df8024..87560d8b254e 100644 --- a/arch/arm64/include/asm/el2_setup.h +++ b/arch/arm64/include/asm/el2_setup.h @@ -418,6 +418,7 @@ b.lt .Lskip_fgt2_\@ mov x0, xzr + mov x2, xzr mrs x1, id_aa64dfr0_el1 ubfx x1, x1, #ID_AA64DFR0_EL1_PMUVer_SHIFT, #4 cmp x1, #ID_AA64DFR0_EL1_PMUVer_V3P9 @@ -426,6 +427,11 @@ orr x0, x0, #HDFGRTR2_EL2_nPMICNTR_EL0 orr x0, x0, #HDFGRTR2_EL2_nPMICFILTR_EL0 orr x0, x0, #HDFGRTR2_EL2_nPMUACR_EL1 + orr x2, x2, #HDFGWTR2_EL2_nPMICNTR_EL0 + orr x2, x2, #HDFGWTR2_EL2_nPMICFILTR_EL0 + orr x2, x2, #HDFGWTR2_EL2_nPMUACR_EL1 + /* PMZR_EL0 is write-only, so it has no read trap to disable */ + orr x2, x2, #HDFGWTR2_EL2_nPMZR_EL0 .Lskip_pmuv3p9_\@: /* If SPE is implemented, */ __spe_vers_imp .Lskip_spefds_\@, ID_AA64DFR0_EL1_PMSVer_IMP, x1 @@ -436,10 +442,11 @@ cbz x1, .Lskip_spefds_\@ /* disable traps of PMSDSFR to EL2. */ orr x0, x0, #HDFGRTR2_EL2_nPMSDSFR_EL1 + orr x2, x2, #HDFGWTR2_EL2_nPMSDSFR_EL1 .Lskip_spefds_\@: msr_s SYS_HDFGRTR2_EL2, x0 - msr_s SYS_HDFGWTR2_EL2, x0 + msr_s SYS_HDFGWTR2_EL2, x2 msr_s SYS_HFGRTR2_EL2, xzr msr_s SYS_HFGWTR2_EL2, xzr msr_s SYS_HFGITR2_EL2, xzr diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h index 49a7002661a9..9aa0bb08ab60 100644 --- a/arch/arm64/include/asm/io.h +++ b/arch/arm64/include/asm/io.h @@ -280,7 +280,8 @@ static inline void __iomem *ioremap_prot(phys_addr_t phys, size_t size, pgprot_t prot; ptval_t user_prot_val = pgprot_val(user_prot); - if (WARN_ON_ONCE(!(user_prot_val & PTE_USER))) + /* Reject PROT_NONE and exec-only */ + if (!(user_prot_val & PTE_USER)) return NULL; prot = __pgprot_modify(PAGE_KERNEL, PTE_ATTRINDX_MASK, diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 27fe0cd5b2d7..cd9b9d2462f9 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -322,7 +322,7 @@ struct kvm_arch { * Stage 2 paging state for VMs with nested S2 using a virtual * VMID. */ - struct kvm_s2_mmu *nested_mmus; + struct kvm_s2_mmu **nested_mmus; size_t nested_mmus_size; int nested_mmus_next; diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h index 1ed708335809..586026e85903 100644 --- a/arch/arm64/include/asm/kvm_nested.h +++ b/arch/arm64/include/asm/kvm_nested.h @@ -66,7 +66,8 @@ static inline u64 translate_ttbr0_el2_to_ttbr0_el1(u64 ttbr0) extern bool forward_smc_trap(struct kvm_vcpu *vcpu); extern bool forward_debug_exception(struct kvm_vcpu *vcpu); -extern void kvm_init_nested(struct kvm *kvm); +extern int kvm_init_nested(struct kvm *kvm); +extern void kvm_destroy_nested(struct kvm *kvm); extern int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu); extern void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu); extern struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu); diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index beea00e693a0..cad60569f061 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -65,8 +65,7 @@ static inline bool kvm_pkvm_ioctl_allowed(struct kvm *kvm, unsigned int ioctl) int r; r = kvm_get_cap_for_kvm_ioctl(ioctl, &ext); - - if (WARN_ON_ONCE(r < 0)) + if (r < 0) return false; return kvm_pkvm_ext_allowed(kvm, ext); diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index b33dccfafaf8..e0c09402540c 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -28,18 +28,21 @@ bool cpu_errata_set_target_impl(u64 num, void *impl_cpus) return true; } +static inline bool __is_midr_in_range(u32 midr, struct midr_range const *range) +{ + return midr_is_cpu_model_range(midr, range->model, + range->rv_min, range->rv_max); +} + static inline bool is_midr_in_range(struct midr_range const *range) { int i; if (!target_impl_cpu_num) - return midr_is_cpu_model_range(read_cpuid_id(), range->model, - range->rv_min, range->rv_max); + return __is_midr_in_range(read_cpuid_id(), range); for (i = 0; i < target_impl_cpu_num; i++) { - if (midr_is_cpu_model_range(target_impl_cpus[i].midr, - range->model, - range->rv_min, range->rv_max)) + if (__is_midr_in_range(target_impl_cpus[i].midr, range)) return true; } return false; @@ -59,7 +62,7 @@ __is_affected_midr_range(const struct arm64_cpu_capabilities *entry, u32 midr, u32 revidr) { const struct arm64_midr_revidr *fix; - if (!is_midr_in_range(&entry->midr_range)) + if (!__is_midr_in_range(midr, &entry->midr_range)) return false; midr &= MIDR_REVISION_MASK | MIDR_VARIANT_MASK; @@ -381,6 +384,20 @@ static const struct arm64_cpu_capabilities arm64_repeat_tlbi_list[] = { }; #endif +#ifdef CONFIG_ARM64_WORKAROUND_BROKEN_AMU_CONSTCNT +static const struct midr_range workaround_amu_constcnt_list[] = { +#ifdef CONFIG_ARM64_ERRATUM_2457168 + /* Cortex-A510 r0p0-r1p1 */ + MIDR_RANGE(MIDR_CORTEX_A510, 0, 0, 1, 1), +#endif +#ifdef CONFIG_ARM64_ERRATUM_3821522 + /* Cortex-A725 r0p0 - r0p2 */ + MIDR_RANGE(MIDR_CORTEX_A725, 0, 0, 0, 2), +#endif + {} +}; +#endif /* CONFIG_ARM64_WORKAROUND_BROKEN_AMU_CONSTCNT */ + #ifdef CONFIG_CAVIUM_ERRATUM_23154 static const struct midr_range cavium_erratum_23154_cpus[] = { MIDR_ALL_VERSIONS(MIDR_THUNDERX), @@ -916,14 +933,12 @@ const struct arm64_cpu_capabilities arm64_errata[] = { ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A510, 0, 0, 2) }, #endif -#ifdef CONFIG_ARM64_ERRATUM_2457168 +#ifdef CONFIG_ARM64_WORKAROUND_BROKEN_AMU_CONSTCNT { - .desc = "ARM erratum 2457168", - .capability = ARM64_WORKAROUND_2457168, + .desc = "Broken AMU AMEVCNTR01 (const counter)", + .capability = ARM64_WORKAROUND_BROKEN_AMU_CONSTCNT, .type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE, - - /* Cortex-A510 r0p0-r1p1 */ - CAP_MIDR_RANGE(MIDR_CORTEX_A510, 0, 0, 1, 1) + CAP_MIDR_RANGE_LIST(workaround_amu_constcnt_list) }, #endif #ifdef CONFIG_ARM64_ERRATUM_2038923 diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 32102c3912fa..3ae3ad1a1cdf 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2089,7 +2089,7 @@ static void cpu_amu_enable(struct arm64_cpu_capabilities const *cap) cpumask_set_cpu(smp_processor_id(), &amu_cpus); /* 0 reference values signal broken/disabled counters */ - if (!this_cpu_has_cap(ARM64_WORKAROUND_2457168)) + if (!this_cpu_has_cap(ARM64_WORKAROUND_BROKEN_AMU_CONSTCNT)) update_freq_counters_refs(); } } diff --git a/arch/arm64/kernel/pi/idreg-override.c b/arch/arm64/kernel/pi/idreg-override.c index 274bc72ba98d..acaf56234973 100644 --- a/arch/arm64/kernel/pi/idreg-override.c +++ b/arch/arm64/kernel/pi/idreg-override.c @@ -186,6 +186,7 @@ static const struct ftr_set_desc isar2 __prel64_initconst = { .name = "id_aa64isar2", .override = &id_aa64isar2_override, .fields = { + FIELD("wfxt", ID_AA64ISAR2_EL1_WFxT_SHIFT, NULL), FIELD("gpa3", ID_AA64ISAR2_EL1_GPA3_SHIFT, NULL), FIELD("apa3", ID_AA64ISAR2_EL1_APA3_SHIFT, NULL), FIELD("mops", ID_AA64ISAR2_EL1_MOPS_SHIFT, NULL), @@ -254,6 +255,7 @@ static const struct { "id_aa64isar1.api=0 id_aa64isar1.apa=0 " "id_aa64isar2.gpa3=0 id_aa64isar2.apa3=0" }, { "arm64.nomops", "id_aa64isar2.mops=0" }, + { "arm64.nowfxt", "id_aa64isar2.wfxt=0" }, { "arm64.nomte", "id_aa64pfr1.mte=0" }, { "nokaslr", "arm64_sw.nokaslr=1" }, { "rodata=off", "arm64_sw.rodataoff=1" }, diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index 38e6fa204c17..6a06a4e352b1 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -433,6 +433,7 @@ static int restore_sve_fpsimd_context(struct user_ctxs *user) unsigned int vl, vq; struct user_fpsimd_state fpsimd; u16 user_vl, flags; + bool fpsimd_only; bool sm; if (user->sve_size < sizeof(*user->sve)) @@ -443,19 +444,33 @@ static int restore_sve_fpsimd_context(struct user_ctxs *user) if (err) return err; + fpsimd_only = (user->sve_size == sizeof(*user->sve)); sm = flags & SVE_SIG_FLAG_SM; + if (sm) { if (!system_supports_sme()) return -EINVAL; + /* + * Streaming SVE state is always preserved with an SVE payload. + * Only accept streaming state which has an SVE payload. + */ + if (fpsimd_only) + return -EINVAL; + vl = task_get_sme_vl(current); } else { /* - * A SME only system use SVE for streaming mode so can - * have a SVE formatted context with a zero VL and no - * payload data. + * Non-streaming SVE state may be preserved without an SVE + * payload, in which case all state is saved in the FPSIMD + * context. + * + * On SME-only systems, non-streaming (FPSIMD-only) state is + * always preserved without an SVE payload, and with VL==0. On + * such systems, only accept non-streaming state without an SVE + * payload. */ - if (!system_supports_sve() && !system_supports_sme()) + if (!system_supports_sve() && !fpsimd_only) return -EINVAL; vl = task_get_sve_vl(current); @@ -464,16 +479,7 @@ static int restore_sve_fpsimd_context(struct user_ctxs *user) if (user_vl != vl) return -EINVAL; - /* - * Non-streaming SVE state may be preserved without an SVE payload, in - * which case the SVE context only has a header with VL==0, and all - * state can be restored from the FPSIMD context. - * - * Streaming SVE state is always preserved with an SVE payload. For - * consistency and robustness, reject restoring streaming SVE state - * without an SVE payload. - */ - if (!sm && user->sve_size == sizeof(*user->sve)) + if (fpsimd_only) return restore_fpsimd_context(user); vq = sve_vq_from_vl(vl); diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c index d28438f8b83f..806373aac3db 100644 --- a/arch/arm64/kernel/topology.c +++ b/arch/arm64/kernel/topology.c @@ -186,7 +186,6 @@ int arch_freq_get_on_cpu(int cpu) struct amu_cntr_sample *amu_sample; unsigned int start_cpu = cpu; unsigned long last_update; - unsigned int freq = 0; u64 scale; if (!amu_fie_cpu_supported(cpu) || !arch_scale_freq_ref(cpu)) @@ -245,9 +244,8 @@ int arch_freq_get_on_cpu(int cpu) * (see amu_scale_freq_tick for details) */ scale = arch_scale_freq_capacity(cpu); - freq = scale * arch_scale_freq_ref(cpu); - freq >>= SCHED_CAPACITY_SHIFT; - return freq; + + return (scale * arch_scale_freq_ref(cpu)) >> SCHED_CAPACITY_SHIFT; } static void amu_fie_setup(const struct cpumask *cpus) @@ -397,12 +395,13 @@ static void cpu_read_corecnt(void *val) static void cpu_read_constcnt(void *val) { /* - * Return 0 if the current CPU is affected by erratum 2457168. A value - * of 0 is also returned if the current CPU does not support AMUs or if - * the counter is disabled. A return value of 0 at counter read is - * properly handled as an error case by the users of the counter. + * Return 0 if the current CPU is affected by a HW erratum. + * A value of 0 is also returned if the current CPU does not + * support AMUs or if the counter is disabled. A return + * value of 0 at counter read is properly handled as an error + * case by the users of the counter. */ - *(u64 *)val = this_cpu_has_cap(ARM64_WORKAROUND_2457168) ? + *(u64 *)val = this_cpu_has_cap(ARM64_WORKAROUND_BROKEN_AMU_CONSTCNT) ? 0UL : read_constcnt(); } @@ -463,8 +462,9 @@ static void amu_read_core_const_ctrs(void *val) /* * cpu_read_constcnt() incurs slight latency due to the - * ARM64_WORKAROUND_2457168 check. Read it first to minimize - * the sampling skew between the const and core counters. + * ARM64_WORKAROUND_BROKEN_AMU_CONSTCNT check. + * Read it first to minimize the sampling skew between the const + * and core counters. */ cpu_read_constcnt(&ctrs->constcnt); cpu_read_corecnt(&ctrs->corecnt); diff --git a/arch/arm64/kernel/vdso/Makefile b/arch/arm64/kernel/vdso/Makefile index 7dec05dd33b7..f6619e1cb2ce 100644 --- a/arch/arm64/kernel/vdso/Makefile +++ b/arch/arm64/kernel/vdso/Makefile @@ -41,7 +41,7 @@ CC_FLAGS_REMOVE_VDSO := $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) \ $(CC_FLAGS_LTO) $(CC_FLAGS_CFI) \ -Wmissing-prototypes -Wmissing-declarations -CC_FLAGS_ADD_VDSO := -O2 -mcmodel=tiny -fasynchronous-unwind-tables +CC_FLAGS_ADD_VDSO := -O2 -mcmodel=tiny -fasynchronous-unwind-tables $(CONFIG_CC_OPT_INLINE_MEMSET) CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_REMOVE_VDSO) CFLAGS_REMOVE_vgetrandom.o = $(CC_FLAGS_REMOVE_VDSO) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 8b080804bc90..0576c2022ef5 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -236,8 +236,6 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) mutex_unlock(&kvm->lock); #endif - kvm_init_nested(kvm); - ret = kvm_share_hyp(kvm, kvm + 1); if (ret) return ret; @@ -252,6 +250,10 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) if (ret) goto err_free_cpumask; + ret = kvm_init_nested(kvm); + if (ret) + goto err_uninit_mmu; + if (is_protected_kvm_enabled()) { /* * If any failures occur after this is successful, make sure to @@ -280,6 +282,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) err_uninit_mmu: kvm_uninit_stage2_mmu(kvm); + kvm_destroy_nested(kvm); err_free_cpumask: free_cpumask_var(kvm->arch.supported_cpus); err_unshare_kvm: @@ -337,6 +340,7 @@ void kvm_arch_destroy_vm(struct kvm *kvm) kvm_unshare_hyp(kvm, kvm + 1); + kvm_destroy_nested(kvm); kvm_arm_teardown_hypercalls(kvm); } @@ -1129,7 +1133,7 @@ static int kvm_vcpu_suspend(struct kvm_vcpu *vcpu) static int check_vcpu_requests(struct kvm_vcpu *vcpu) { if (kvm_request_pending(vcpu)) { - if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) + if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu)) return -EIO; if (kvm_check_request(KVM_REQ_SLEEP, vcpu)) diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c index 625604019fb3..f59d89d04eca 100644 --- a/arch/arm64/kvm/emulate-nested.c +++ b/arch/arm64/kvm/emulate-nested.c @@ -1445,7 +1445,7 @@ static const struct encoding_to_trap_config encoding_to_fgt[] __initconst = { SR_FGT(OP_AT_S1E1A, HFGITR, ATS1E1A, 1), SR_FGT(OP_COSP_RCTX, HFGITR, COSPRCTX, 1), SR_FGT(OP_GCSPUSHX, HFGITR, nGCSEPP, 0), - SR_FGT(OP_GCSPOPX, HFGITR, nGCSEPP, 0), + SR_FGT(OP_GCSPOPCX, HFGITR, nGCSEPP, 0), SR_FGT(OP_GCSPUSHM, HFGITR, nGCSPUSHM_EL1, 0), SR_FGT(OP_BRB_IALL, HFGITR, nBRBIALL, 0), SR_FGT(OP_BRB_INJ, HFGITR, nBRBINJ, 0), @@ -2386,9 +2386,6 @@ int __init populate_nv_trap_config(void) print_nv_trap_error(fgt, "FGT bit is reserved", ret); } - if (!cpus_have_final_cap(ARM64_HAS_FGT)) - continue; - prev = xa_store(&sr_forward_xa, enc, xa_mk_value(tc.val), GFP_KERNEL); diff --git a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h index 29935c7da1de..ec85a9547120 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h @@ -52,8 +52,10 @@ int __pkvm_host_test_clear_young_guest(u64 gfn, u64 nr_pages, bool mkold, struct int __pkvm_host_mkyoung_guest(u64 gfn, struct pkvm_hyp_vcpu *vcpu); bool addr_is_memory(phys_addr_t phys); +bool addr_is_hyp_text(phys_addr_t phys); int host_stage2_idmap_locked(phys_addr_t addr, u64 size, enum kvm_pgtable_prot prot); int host_stage2_set_owner_locked(phys_addr_t addr, u64 size, u8 owner_id); +bool host_stage2_pte_is_hyp_owned(kvm_pte_t pte); int kvm_host_prepare_stage2(void *pgt_pool_base); int kvm_guest_prepare_stage2(struct pkvm_hyp_vm *vm, void *pgd); void kvm_guest_destroy_stage2(struct pkvm_hyp_vm *vm); diff --git a/arch/arm64/kvm/hyp/include/nvhe/mm.h b/arch/arm64/kvm/hyp/include/nvhe/mm.h index 6e83ce35c2f2..31cae95ddb71 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mm.h @@ -29,6 +29,7 @@ int __pkvm_create_private_mapping(phys_addr_t phys, size_t size, enum kvm_pgtable_prot prot, unsigned long *haddr); int pkvm_create_stack(phys_addr_t phys, unsigned long *haddr); +int pkvm_check_host_ownership(void); int pkvm_alloc_private_va_range(size_t size, unsigned long *haddr); #endif /* __KVM_HYP_MM_H */ diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h index c904647d2f76..5c050f21066a 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h @@ -12,6 +12,15 @@ #include #include +/* + * HCR_EL2 bits EL2 takes from the host on each entry, per VM type. The rest + * are EL2's own and nothing the host sets there reaches the guest. + */ +#define PKVM_HCR_EL2_HOST_PVM (HCR_EL2_TWI | HCR_EL2_TWE | HCR_EL2_VSE) +#define PKVM_HCR_EL2_HOST_NPVM (PKVM_HCR_EL2_HOST_PVM | HCR_EL2_VI | HCR_EL2_VF | \ + HCR_EL2_TVM | HCR_EL2_TID2 | HCR_EL2_TID4 | \ + HCR_EL2_TID5 | HCR_EL2_TTLBOS) + /* * Holds the relevant data for maintaining the vcpu state completely at hyp. */ diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 9a3b92e626ad..ac64a036b0a9 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -216,6 +216,7 @@ static void sync_debug_state(struct pkvm_hyp_vcpu *hyp_vcpu) static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + u64 host_hcr_mask = PKVM_HCR_EL2_HOST_PVM; fpsimd_sve_flush(); flush_debug_state(hyp_vcpu); @@ -228,6 +229,7 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { if (vcpu_get_flag(host_vcpu, PKVM_HOST_STATE_DIRTY)) flush_hyp_vcpu_state(hyp_vcpu); + host_hcr_mask = PKVM_HCR_EL2_HOST_NPVM; } else { hyp_vcpu->vcpu.arch.ctxt = host_vcpu->arch.ctxt; } @@ -241,9 +243,8 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) * trap-control bit, so it must flow to the hyp vCPU alongside TWI/TWE * for the vSError to be delivered. sync_hyp_vcpu() reflects it back. */ - hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_TWI | HCR_TWE | HCR_VSE); - hyp_vcpu->vcpu.arch.hcr_el2 |= READ_ONCE(host_vcpu->arch.hcr_el2) & - (HCR_TWI | HCR_TWE | HCR_VSE); + hyp_vcpu->vcpu.arch.hcr_el2 &= ~host_hcr_mask; + hyp_vcpu->vcpu.arch.hcr_el2 |= READ_ONCE(host_vcpu->arch.hcr_el2) & host_hcr_mask; hyp_vcpu->vcpu.arch.iflags = host_vcpu->arch.iflags; diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 39aa8911f62c..a6a47c1e058b 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -450,6 +450,14 @@ bool addr_is_memory(phys_addr_t phys) return !!find_mem_range(phys, &range); } +bool addr_is_hyp_text(phys_addr_t phys) +{ + phys_addr_t start = ALIGN_DOWN(__hyp_pa(__hyp_text_start), PAGE_SIZE); + phys_addr_t end = PAGE_ALIGN(__hyp_pa(__hyp_text_end)); + + return phys >= start && phys < end; +} + static bool is_in_mem_range(u64 addr, struct kvm_mem_range *range) { return range->start <= addr && addr < range->end; @@ -633,6 +641,18 @@ int host_stage2_set_owner_locked(phys_addr_t addr, u64 size, u8 owner_id) return ret; } +bool host_stage2_pte_is_hyp_owned(kvm_pte_t pte) +{ + if (kvm_pte_valid(pte)) + return false; + + if (FIELD_GET(KVM_INVALID_PTE_TYPE_MASK, pte) != + KVM_HOST_INVALID_PTE_TYPE_DONATION) + return false; + + return FIELD_GET(KVM_HOST_DONATION_PTE_OWNER_MASK, pte) == PKVM_ID_HYP; +} + #define KVM_HOST_PTE_OWNER_GUEST_HANDLE_MASK GENMASK(15, 0) /* We need 40 bits for the GFN to cover a 52-bit IPA with 4k pages and LPA2 */ #define KVM_HOST_PTE_OWNER_GUEST_GFN_MASK GENMASK(55, 16) diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c index 3b0bee496bff..29ab5ee9d57f 100644 --- a/arch/arm64/kvm/hyp/nvhe/mm.c +++ b/arch/arm64/kvm/hyp/nvhe/mm.c @@ -25,6 +25,7 @@ struct memblock_region hyp_memory[HYP_MEMBLOCK_REGIONS]; unsigned int hyp_memblock_nr; static u64 __io_map_base; +static u64 __io_map_next; struct hyp_fixmap_slot { u64 addr; @@ -50,7 +51,7 @@ static int __pkvm_alloc_private_va_range(unsigned long start, size_t size) hyp_assert_lock_held(&pkvm_pgd_lock); - if (!start || start < __io_map_base) + if (!start || start < __io_map_next) return -EINVAL; /* The allocated size is always a multiple of PAGE_SIZE */ @@ -60,7 +61,7 @@ static int __pkvm_alloc_private_va_range(unsigned long start, size_t size) if (cur > __hyp_vmemmap) return -ENOMEM; - __io_map_base = cur; + __io_map_next = cur; return 0; } @@ -70,7 +71,7 @@ static int __pkvm_alloc_private_va_range(unsigned long start, size_t size) * @size: The size of the VA range to reserve. * @haddr: The hypervisor virtual start address of the allocation. * - * The private virtual address (VA) range is allocated above __io_map_base + * The private virtual address (VA) range is allocated above __io_map_next * and aligned based on the order of @size. * * Return: 0 on success or negative error code on failure. @@ -81,7 +82,7 @@ int pkvm_alloc_private_va_range(size_t size, unsigned long *haddr) int ret; hyp_spin_lock(&pkvm_pgd_lock); - addr = __io_map_base; + addr = __io_map_next; ret = __pkvm_alloc_private_va_range(addr, size); hyp_spin_unlock(&pkvm_pgd_lock); @@ -341,7 +342,7 @@ static int create_fixblock(void) return -EINVAL; hyp_spin_lock(&pkvm_pgd_lock); - addr = ALIGN(__io_map_base, PMD_SIZE); + addr = ALIGN(__io_map_next, PMD_SIZE); ret = __pkvm_alloc_private_va_range(addr, PMD_SIZE); if (ret) goto unlock; @@ -426,6 +427,7 @@ int hyp_create_idmap(u32 hyp_va_bits) */ __io_map_base = start & BIT(hyp_va_bits - 2); __io_map_base ^= BIT(hyp_va_bits - 2); + __io_map_next = __io_map_base; __hyp_vmemmap = __io_map_base | BIT(hyp_va_bits - 3); return __pkvm_create_mappings(start, end - start, start, PAGE_HYP_EXEC); @@ -433,19 +435,19 @@ int hyp_create_idmap(u32 hyp_va_bits) int pkvm_create_stack(phys_addr_t phys, unsigned long *haddr) { - unsigned long addr, prev_base; + unsigned long addr, prev_next; size_t size; int ret; hyp_spin_lock(&pkvm_pgd_lock); - prev_base = __io_map_base; + prev_next = __io_map_next; /* * Efficient stack verification using the NVHE_STACK_SHIFT bit implies * an alignment of our allocation on the order of the size. */ size = NVHE_STACK_SIZE * 2; - addr = ALIGN(__io_map_base, size); + addr = ALIGN(__io_map_next, size); ret = __pkvm_alloc_private_va_range(addr, size); if (!ret) { @@ -461,7 +463,7 @@ int pkvm_create_stack(phys_addr_t phys, unsigned long *haddr) ret = kvm_pgtable_hyp_map(&pkvm_pgtable, addr + NVHE_STACK_SIZE, NVHE_STACK_SIZE, phys, PAGE_HYP); if (ret) - __io_map_base = prev_base; + __io_map_next = prev_next; } hyp_spin_unlock(&pkvm_pgd_lock); @@ -470,6 +472,65 @@ int pkvm_create_stack(phys_addr_t phys, unsigned long *haddr) return ret; } +static int check_page_ownership(phys_addr_t phys) +{ + kvm_pte_t pte; + bool host_ok; + int ret; + + if (addr_is_memory(phys)) { + struct hyp_page *page = hyp_phys_to_page(phys); + + if (get_hyp_state(page) != PKVM_PAGE_OWNED || + get_host_state(page) != PKVM_NOPAGE) + return -EPERM; + } + + ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, NULL); + if (ret) + return ret; + + /* Hyp text may stay host-readable, see fix_host_ownership_walker(). */ + if (kvm_pte_valid(pte) && addr_is_hyp_text(phys)) + host_ok = !(kvm_pgtable_stage2_pte_prot(pte) & KVM_PGTABLE_PROT_W); + else + host_ok = host_stage2_pte_is_hyp_owned(pte); + + return host_ok ? 0 : -EPERM; +} + +static int check_host_ownership_walker(const struct kvm_pgtable_visit_ctx *ctx, + enum kvm_pgtable_walk_flags visit) +{ + phys_addr_t phys, end; + int ret; + + if (!kvm_pte_valid(ctx->old)) + return 0; + + phys = kvm_pte_to_phys(ctx->old); + end = phys + kvm_granule_size(ctx->level); + for (; phys < end; phys += PAGE_SIZE) { + ret = check_page_ownership(phys); + if (ret) + return ret; + } + + return 0; +} + +int pkvm_check_host_ownership(void) +{ + struct kvm_pgtable_walker walker = { + .cb = check_host_ownership_walker, + .flags = KVM_PGTABLE_WALK_LEAF, + }; + + /* The private range and the vmemmap share one quarter of the VA space. */ + return kvm_pgtable_walk(&pkvm_pgtable, __io_map_base, + BIT(pkvm_pgtable.ia_bits - 2), &walker); +} + static void *admit_host_page(void *arg) { struct kvm_hyp_memcache *host_mc = arg; diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 459bd9eb7e4b..6290c4b62659 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -30,6 +30,7 @@ unsigned int kvm_host_sve_max_vl; */ static DEFINE_PER_CPU(struct pkvm_hyp_vcpu *, loaded_hyp_vcpu); +/* The PKVM_HCR_EL2_HOST_{PVM,NPVM} bits of this value come from the host on each entry. */ static void pkvm_vcpu_reset_hcr(struct kvm_vcpu *vcpu) { vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS; @@ -47,18 +48,17 @@ static void pkvm_vcpu_reset_hcr(struct kvm_vcpu *vcpu) if (cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) vcpu->arch.hcr_el2 |= HCR_FWB; - if (cpus_have_final_cap(ARM64_HAS_EVT) && - !cpus_have_final_cap(ARM64_MISMATCHED_CACHE_TYPE) && - kvm_read_vm_id_reg(vcpu->kvm, SYS_CTR_EL0) == read_cpuid(CTR_EL0)) - vcpu->arch.hcr_el2 |= HCR_TID4; - else - vcpu->arch.hcr_el2 |= HCR_TID2; + /* + * Without AArch32 EL1, leave RW set and let the entry fail with an + * illegal exception return: the *32_EL2 registers EL2 would otherwise + * switch are UNDEFINED there. + */ + if (vcpu_has_feature(vcpu, KVM_ARM_VCPU_EL1_32BIT) && + cpus_have_final_cap(ARM64_HAS_32BIT_EL1)) + vcpu->arch.hcr_el2 &= ~HCR_EL2_RW; if (vcpu_has_ptrauth(vcpu)) vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK); - - if (kvm_has_mte(vcpu->kvm)) - vcpu->arch.hcr_el2 |= HCR_ATA; } static void pvm_init_traps_hcr(struct kvm_vcpu *vcpu) @@ -76,6 +76,13 @@ static void pvm_init_traps_hcr(struct kvm_vcpu *vcpu) */ val |= HCR_TACR | HCR_TIDCP | HCR_TID3 | HCR_TID1; + if (cpus_have_final_cap(ARM64_HAS_EVT) && + !cpus_have_final_cap(ARM64_MISMATCHED_CACHE_TYPE) && + kvm_read_vm_id_reg(kvm, SYS_CTR_EL0) == read_cpuid(CTR_EL0)) + val |= HCR_EL2_TID4; + else + val |= HCR_EL2_TID2; + if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, IMP)) { val |= HCR_TERR | HCR_TEA; val &= ~(HCR_FIEN); @@ -360,7 +367,7 @@ static void pkvm_init_features_from_host(struct pkvm_hyp_vm *hyp_vm, const struc if (test_bit(KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS, &host_arch_flags)) hyp_vm->kvm.arch.midr_el1 = host_kvm->arch.midr_el1; - return; + goto out; } if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_MTE)) @@ -379,13 +386,14 @@ static void pkvm_init_features_from_host(struct pkvm_hyp_vm *hyp_vm, const struc if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PTRAUTH_GENERIC)) set_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, allowed_features); - if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_SVE)) { + if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_SVE)) set_bit(KVM_ARM_VCPU_SVE, allowed_features); - kvm->arch.flags |= host_arch_flags & BIT(KVM_ARCH_FLAG_GUEST_HAS_SVE); - } bitmap_and(kvm->arch.vcpu_features, host_kvm->arch.vcpu_features, allowed_features, KVM_VCPU_MAX_FEATURES); +out: + __assign_bit(KVM_ARCH_FLAG_GUEST_HAS_SVE, &kvm->arch.flags, + kvm_vcpu_has_feature(kvm, KVM_ARM_VCPU_SVE)); } static void unpin_host_vcpu(struct kvm_vcpu *host_vcpu) @@ -398,10 +406,10 @@ static void unpin_host_sve_state(struct pkvm_hyp_vcpu *hyp_vcpu) { void *sve_state; - if (!vcpu_has_feature(&hyp_vcpu->vcpu, KVM_ARM_VCPU_SVE)) + sve_state = hyp_vcpu->vcpu.arch.sve_state; + if (!sve_state) return; - sve_state = hyp_vcpu->vcpu.arch.sve_state; hyp_unpin_shared_mem(sve_state, sve_state + vcpu_sve_state_size(&hyp_vcpu->vcpu)); } @@ -450,7 +458,7 @@ static int pkvm_vcpu_init_sve(struct pkvm_hyp_vcpu *hyp_vcpu, struct kvm_vcpu *h unsigned int sve_max_vl; size_t sve_state_size; void *sve_state; - int ret = 0; + int ret; if (!vcpu_has_feature(vcpu, KVM_ARM_VCPU_SVE)) { vcpu_clear_flag(vcpu, VCPU_SVE_FINALIZED); @@ -459,25 +467,21 @@ static int pkvm_vcpu_init_sve(struct pkvm_hyp_vcpu *hyp_vcpu, struct kvm_vcpu *h /* Limit guest vector length to the maximum supported by the host. */ sve_max_vl = min(READ_ONCE(host_vcpu->arch.sve_max_vl), kvm_host_sve_max_vl); - sve_state_size = sve_state_size_from_vl(sve_max_vl); sve_state = kern_hyp_va(READ_ONCE(host_vcpu->arch.sve_state)); - if (!sve_state || !sve_state_size) { - ret = -EINVAL; - goto err; - } + if (!sve_vl_valid(sve_max_vl) || !sve_state) + return -EINVAL; + + sve_state_size = sve_state_size_from_vl(sve_max_vl); ret = hyp_pin_shared_mem(sve_state, sve_state + sve_state_size); if (ret) - goto err; + return ret; vcpu->arch.sve_state = sve_state; vcpu->arch.sve_max_vl = sve_max_vl; return 0; -err: - clear_bit(KVM_ARM_VCPU_SVE, vcpu->kvm->arch.vcpu_features); - return ret; } static int vm_copy_id_regs(struct pkvm_hyp_vcpu *hyp_vcpu) diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c index 75b00c323310..45ac5f2ba4f7 100644 --- a/arch/arm64/kvm/hyp/nvhe/setup.c +++ b/arch/arm64/kvm/hyp/nvhe/setup.c @@ -217,7 +217,7 @@ static int fix_host_ownership_walker(const struct kvm_pgtable_visit_ctx *ctx, case PKVM_PAGE_OWNED: set_hyp_state(page, PKVM_PAGE_OWNED); /* hyp text is RO in the host stage-2 to be inspected on panic. */ - if (prot == PAGE_HYP_EXEC) { + if (addr_is_hyp_text(phys)) { set_host_state(page, PKVM_NOPAGE); return host_stage2_idmap_locked(phys, PAGE_SIZE, KVM_PGTABLE_PROT_R); } else { @@ -269,6 +269,16 @@ static int fix_host_ownership(void) return ret; } + /* The stacks sit in the private VA range, not the linear map. */ + for (i = 0; i < hyp_nr_cpus; i++) { + struct kvm_nvhe_init_params *params = per_cpu_ptr(&kvm_init_params, i); + u64 start = params->stack_hyp_va - NVHE_STACK_SIZE; + + ret = kvm_pgtable_walk(&pkvm_pgtable, start, NVHE_STACK_SIZE, &walker); + if (ret) + return ret; + } + return 0; } @@ -324,6 +334,10 @@ void __noreturn __pkvm_init_finalise(void) if (ret) goto out; + ret = pkvm_check_host_ownership(); + if (ret) + goto out; + ret = hyp_ffa_init(ffa_proxy_pages); if (ret) goto out; diff --git a/arch/arm64/kvm/hypercalls.c b/arch/arm64/kvm/hypercalls.c index b11b8821c9fb..dfa25bb6f25d 100644 --- a/arch/arm64/kvm/hypercalls.c +++ b/arch/arm64/kvm/hypercalls.c @@ -185,7 +185,8 @@ static int kvm_smccc_set_filter(struct kvm *kvm, struct kvm_smccc_filter __user start = filter.base; end = start + filter.nr_functions - 1; - if (end < start || filter.action >= NR_SMCCC_FILTER_ACTIONS) + if (!filter.nr_functions || end < start || + filter.action >= NR_SMCCC_FILTER_ACTIONS) return -EINVAL; mutex_lock(&kvm->arch.config_lock); diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 9ba86450fe4a..bf6d6526639f 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -59,27 +59,36 @@ static phys_addr_t stage2_range_addr_end(phys_addr_t addr, phys_addr_t end) * long will also starve other vCPUs. We have to also make sure that the page * tables are not freed while we released the lock. */ -static int stage2_apply_range(struct kvm_s2_mmu *mmu, phys_addr_t addr, +static int stage2_apply_range(struct kvm_s2_mmu *mmu, phys_addr_t start, phys_addr_t end, int (*fn)(struct kvm_pgtable *, u64, u64), bool resched) { struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); + bool lock_dropped = false; + phys_addr_t addr = start; int ret; u64 next; do { struct kvm_pgtable *pgt = mmu->pgt; + /* + * We may be raced on PGT teardown when we release the + * kvm->mmu_lock. That's fine as the PGT is legitimately no + * longer present. + */ if (!pgt) - return -EINVAL; + return lock_dropped ? 0 : -EINVAL; next = stage2_range_addr_end(addr, end); ret = fn(pgt, addr, next - addr); if (ret) break; - if (resched && next != end) + if (resched && next != end) { cond_resched_rwlock_write(&kvm->mmu_lock); + lock_dropped = true; + } } while (addr = next, addr != end); return ret; @@ -1459,32 +1468,11 @@ transparent_hugepage_adjust(struct kvm *kvm, struct kvm_memory_slot *memslot, return PAGE_SIZE; } -static int get_vma_page_shift(struct vm_area_struct *vma, unsigned long hva) +static int get_vma_page_shift(struct vm_area_struct *vma) { - unsigned long pa; - - if (is_vm_hugetlb_page(vma) && !(vma->vm_flags & VM_PFNMAP)) + if (is_vm_hugetlb_page(vma)) return huge_page_shift(hstate_vma(vma)); - if (!(vma->vm_flags & VM_PFNMAP)) - return PAGE_SHIFT; - - VM_BUG_ON(is_vm_hugetlb_page(vma)); - - pa = (vma->vm_pgoff << PAGE_SHIFT) + (hva - vma->vm_start); - -#ifndef __PAGETABLE_PMD_FOLDED - if ((hva & (PUD_SIZE - 1)) == (pa & (PUD_SIZE - 1)) && - ALIGN_DOWN(hva, PUD_SIZE) >= vma->vm_start && - ALIGN(hva, PUD_SIZE) <= vma->vm_end) - return PUD_SHIFT; -#endif - - if ((hva & (PMD_SIZE - 1)) == (pa & (PMD_SIZE - 1)) && - ALIGN_DOWN(hva, PMD_SIZE) >= vma->vm_start && - ALIGN(hva, PMD_SIZE) <= vma->vm_end) - return PMD_SHIFT; - return PAGE_SHIFT; } @@ -1785,7 +1773,7 @@ static short kvm_s2_resolve_vma_size(const struct kvm_s2_fault_desc *s2fd, vma_shift = PAGE_SHIFT; } else { s2vi->max_map_size = PUD_SIZE; - vma_shift = get_vma_page_shift(vma, s2fd->hva); + vma_shift = get_vma_page_shift(vma); } switch (vma_shift) { @@ -1943,16 +1931,6 @@ static int kvm_s2_fault_pin_pfn(const struct kvm_s2_fault_desc *s2fd, return -EFAULT; } } else { - /* - * If the page was identified as device early by looking at - * the VMA flags, vma_pagesize is already representing the - * largest quantity we can map. If instead it was mapped - * via __kvm_faultin_pfn(), vma_pagesize is set to PAGE_SIZE - * and must not be upgraded. - * - * In both cases, we don't let transparent_hugepage_adjust() - * change things at the last minute. - */ s2vi->map_non_cacheable = true; } @@ -2042,10 +2020,10 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd, /* * If we are not forced to use page mapping, check if we are - * backed by a THP and thus use block mapping if possible. + * backed by a huge stage-1 mapping and thus use block mapping if + * possible. */ - if (mapping_size == PAGE_SIZE && - !(s2vi->max_map_size == PAGE_SIZE || s2vi->map_non_cacheable)) { + if (mapping_size == PAGE_SIZE && s2vi->max_map_size != PAGE_SIZE) { if (perm_fault_granule > PAGE_SIZE) { mapping_size = perm_fault_granule; } else { @@ -2126,10 +2104,6 @@ static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd) return ret; } - /* - * Let's check if we will get back a huge page backed by hugetlbfs, or - * get block mapping for device MMIO region. - */ ret = kvm_s2_fault_pin_pfn(s2fd, &s2vi); if (ret != 1) return ret; diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 3c4fc566eafc..b191365d97cc 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -45,11 +45,24 @@ struct vncr_tlb { */ #define S2_MMU_PER_VCPU 2 -void kvm_init_nested(struct kvm *kvm) +int kvm_init_nested(struct kvm *kvm) { - kvm->arch.nested_mmus = NULL; + kvm->arch.nested_mmus = kvmalloc_objs(struct kvm_s2_mmu *, + KVM_MAX_VCPUS * S2_MMU_PER_VCPU, + GFP_KERNEL_ACCOUNT); kvm->arch.nested_mmus_size = 0; atomic_set(&kvm->arch.vncr_tlb_count, 0); + + return kvm->arch.nested_mmus ? 0 : -ENOMEM; +} + +void kvm_destroy_nested(struct kvm *kvm) +{ + for (int i = 0; i < kvm->arch.nested_mmus_size; i+= S2_MMU_PER_VCPU) + kvfree(kvm->arch.nested_mmus[i]); + + kvm->arch.nested_mmus_size = 0; + kvfree(kvm->arch.nested_mmus); } static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) @@ -70,8 +83,9 @@ static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu) { struct kvm *kvm = vcpu->kvm; - struct kvm_s2_mmu *tmp; - int num_mmus, ret = 0; + int num_mmus; + + lockdep_assert_held(&kvm->arch.config_lock); if (test_bit(KVM_ARM_VCPU_HAS_EL2_E2H0, kvm->arch.vcpu_features) && !cpus_have_final_cap(ARM64_HAS_HCR_NV1)) @@ -84,51 +98,40 @@ int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu) if (!vcpu->arch.ctxt.vncr_array) return -ENOMEM; - /* - * Let's treat memory allocation failures as benign: If we fail to - * allocate anything, return an error and keep the allocated array - * alive. Userspace may try to recover by initializing the vcpu - * again, and there is no reason to affect the whole VM for this. - */ num_mmus = atomic_read(&kvm->online_vcpus) * S2_MMU_PER_VCPU; if (num_mmus > kvm->arch.nested_mmus_size) { - tmp = kvzalloc_objs(*tmp, num_mmus, GFP_KERNEL_ACCOUNT); + struct kvm_s2_mmu *tmp; + int i, ret = 0; + + tmp = kvzalloc_objs(*tmp, S2_MMU_PER_VCPU, GFP_KERNEL_ACCOUNT); if (!tmp) - return -ENOMEM; + ret = -ENOMEM; - write_lock(&kvm->mmu_lock); - - if (kvm->arch.nested_mmus_size) { - memcpy(tmp, kvm->arch.nested_mmus, - size_mul(sizeof(*tmp), kvm->arch.nested_mmus_size)); - - for (int i = 0; i < kvm->arch.nested_mmus_size; i++) - tmp[i].pgt->mmu = &tmp[i]; + for (i = 0; !ret && i < S2_MMU_PER_VCPU; i++) { + ret = init_nested_s2_mmu(kvm, &tmp[i]); + if (ret) + break; } - swap(kvm->arch.nested_mmus, tmp); + if (ret) { + while (--i >= 0) + kvm_free_stage2_pgd(&tmp[i]); - write_unlock(&kvm->mmu_lock); + kvfree(tmp); + free_page((unsigned long)vcpu->arch.ctxt.vncr_array); + vcpu->arch.ctxt.vncr_array = NULL; + return ret; + } - kvfree(tmp); + guard(write_lock)(&kvm->mmu_lock); + + for (i = 0; i < S2_MMU_PER_VCPU; i++) + kvm->arch.nested_mmus[i + kvm->arch.nested_mmus_size] = &tmp[i]; + + kvm->arch.nested_mmus_size += S2_MMU_PER_VCPU; } - for (int i = kvm->arch.nested_mmus_size; !ret && i < num_mmus; i++) - ret = init_nested_s2_mmu(kvm, &kvm->arch.nested_mmus[i]); - - if (ret) { - for (int i = kvm->arch.nested_mmus_size; i < num_mmus; i++) - kvm_free_stage2_pgd(&kvm->arch.nested_mmus[i]); - - free_page((unsigned long)vcpu->arch.ctxt.vncr_array); - vcpu->arch.ctxt.vncr_array = NULL; - - return ret; - } - - kvm->arch.nested_mmus_size = num_mmus; - return 0; } @@ -742,7 +745,7 @@ void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid, write_lock(&kvm->mmu_lock); for (int i = 0; i < kvm->arch.nested_mmus_size; i++) { - struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; + struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i]; if (!kvm_s2_mmu_valid(mmu)) continue; @@ -784,7 +787,7 @@ struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu) * if S2 translation is disabled. */ for (int i = 0; i < kvm->arch.nested_mmus_size; i++) { - struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; + struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i]; if (!kvm_s2_mmu_valid(mmu)) continue; @@ -823,7 +826,7 @@ static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu) for (i = kvm->arch.nested_mmus_next; i < (kvm->arch.nested_mmus_size + kvm->arch.nested_mmus_next); i++) { - s2_mmu = &kvm->arch.nested_mmus[i % kvm->arch.nested_mmus_size]; + s2_mmu = kvm->arch.nested_mmus[i % kvm->arch.nested_mmus_size]; if (atomic_read(&s2_mmu->refcnt) == 0) break; @@ -1260,6 +1263,17 @@ void kvm_handle_s1e2_tlbi(struct kvm_vcpu *vcpu, u32 inst, u64 val) invalidate_vncr_va(vcpu->kvm, &scope); } +static void kvm_invalidate_vncr_ipa_all(struct kvm *kvm) +{ + struct kvm_pgtable *pgt = kvm->arch.mmu.pgt; + + lockdep_assert_held_write(&kvm->mmu_lock); + + /* if the mmu lock was dropped, pgt teardown may have raced. */ + if (pgt) + kvm_invalidate_vncr_ipa(kvm, 0, BIT(pgt->ia_bits)); +} + void kvm_nested_s2_wp(struct kvm *kvm) { int i; @@ -1270,13 +1284,13 @@ void kvm_nested_s2_wp(struct kvm *kvm) return; for (i = 0; i < kvm->arch.nested_mmus_size; i++) { - struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; + struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i]; if (kvm_s2_mmu_valid(mmu)) kvm_stage2_wp_range(mmu, 0, kvm_phys_size(mmu)); } - kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits)); + kvm_invalidate_vncr_ipa_all(kvm); } void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block) @@ -1289,13 +1303,13 @@ void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block) return; for (i = 0; i < kvm->arch.nested_mmus_size; i++) { - struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; + struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i]; if (kvm_s2_mmu_valid(mmu)) kvm_stage2_unmap_range(mmu, 0, kvm_phys_size(mmu), may_block); } - kvm_invalidate_vncr_ipa(kvm, 0, BIT(kvm->arch.mmu.pgt->ia_bits)); + kvm_invalidate_vncr_ipa_all(kvm); } void kvm_nested_s2_flush(struct kvm *kvm) @@ -1308,7 +1322,7 @@ void kvm_nested_s2_flush(struct kvm *kvm) return; for (i = 0; i < kvm->arch.nested_mmus_size; i++) { - struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; + struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i]; if (kvm_s2_mmu_valid(mmu)) kvm_stage2_flush_range(mmu, 0, kvm_phys_size(mmu)); @@ -1317,17 +1331,12 @@ void kvm_nested_s2_flush(struct kvm *kvm) void kvm_arch_flush_shadow_all(struct kvm *kvm) { - int i; - - for (i = 0; i < kvm->arch.nested_mmus_size; i++) { - struct kvm_s2_mmu *mmu = &kvm->arch.nested_mmus[i]; + for (int i = 0; i < kvm->arch.nested_mmus_size; i++) { + struct kvm_s2_mmu *mmu = kvm->arch.nested_mmus[i]; if (!WARN_ON(atomic_read(&mmu->refcnt))) kvm_free_stage2_pgd(mmu); } - kvfree(kvm->arch.nested_mmus); - kvm->arch.nested_mmus = NULL; - kvm->arch.nested_mmus_size = 0; kvm_uninit_stage2_mmu(kvm); } diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 44aae52c473d..a2f4e769a428 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -926,6 +926,7 @@ static u64 *demux_wb_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd) struct kvm_guest_debug_arch *dbg = &vcpu->arch.vcpu_debug_state; switch (rd->Op2) { + case 0b001: case 0b100: return &dbg->dbg_bvr[rd->CRm]; case 0b101: diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index 4012df6002ea..a58575df36e9 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -97,6 +97,9 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) /* * - Acquiring the vCPU mutex for every *online* vCPU to prevent * concurrent vCPU ioctls for vCPUs already visible to userspace. + * This also ensures KVM isn't in the middle of creating a vCPU, + * i.e. that there are no vCPUs that have been created but aren't + * yet fully online. */ ret = -EBUSY; if (kvm_trylock_all_vcpus(kvm)) @@ -105,18 +108,11 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) /* * - Taking the config_lock which protects VGIC data structures such * as the per-vCPU arrays of private IRQs (SGIs, PPIs). - */ - mutex_lock(&kvm->arch.config_lock); - - /* - * - Bailing on the entire thing if a vCPU is in the middle of creation, - * dropped the kvm->lock, but hasn't reached kvm_arch_vcpu_create(). * * The whole combination of this guarantees that no vCPU can get into * KVM with a VGIC configuration inconsistent with the VM's VGIC. */ - if (kvm->created_vcpus != atomic_read(&kvm->online_vcpus)) - goto out_unlock; + mutex_lock(&kvm->arch.config_lock); if (irqchip_in_kernel(kvm)) { ret = -EEXIST; diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 9e782a4fea7e..c10a11373e10 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -319,12 +319,16 @@ static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq, return ret; } -static int update_affinity(struct vgic_irq *irq, struct kvm_vcpu *vcpu) +static int update_affinity(struct vgic_irq *irq, + struct kvm_vcpu *from_vcpu, struct kvm_vcpu *vcpu) { struct its_vlpi_map map; int ret; guard(raw_spinlock_irqsave)(&irq->irq_lock); + if (from_vcpu && irq->target_vcpu != from_vcpu) + return 0; + irq->target_vcpu = vcpu; if (!irq->hw) @@ -362,7 +366,7 @@ static void update_affinity_ite(struct kvm *kvm, struct its_ite *ite) return; vcpu = collection_to_vcpu(kvm, ite->collection); - update_affinity(ite->irq, vcpu); + update_affinity(ite->irq, NULL, vcpu); } /* @@ -856,7 +860,7 @@ static int vgic_its_cmd_handle_movi(struct kvm *kvm, struct vgic_its *its, vgic_its_invalidate_cache(its); - return update_affinity(ite->irq, vcpu); + return update_affinity(ite->irq, NULL, vcpu); } static bool __is_visible_gfn_locked(struct vgic_its *its, gpa_t gpa) @@ -1383,7 +1387,7 @@ static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its, if (!irq) continue; - update_affinity(irq, vcpu2); + update_affinity(irq, vcpu1, vcpu2); vgic_put_irq(kvm, irq); } @@ -1658,7 +1662,7 @@ static void vgic_mmio_write_its_baser(struct kvm *kvm, unsigned long val) { const struct vgic_its_abi *abi = vgic_its_get_abi(its); - u64 entry_size, table_type; + u64 old, entry_size, table_type; u64 reg, *regptr, clearbits = 0; /* When GITS_CTLR.Enable is 1, we ignore write accesses. */ @@ -1681,7 +1685,9 @@ static void vgic_mmio_write_its_baser(struct kvm *kvm, return; } - reg = update_64bit_reg(*regptr, addr & 7, len, val); + old = *regptr; + + reg = update_64bit_reg(old, addr & 7, len, val); reg &= ~GITS_BASER_RO_MASK; reg &= ~clearbits; @@ -1691,7 +1697,8 @@ static void vgic_mmio_write_its_baser(struct kvm *kvm, *regptr = reg; - if (!(reg & GITS_BASER_VALID)) { + /* The ITS driver rewrites an unchanged GITS_BASER on resume. */ + if (reg != old) { /* Take the its_lock to prevent a race with a save/restore */ mutex_lock(&its->its_lock); switch (table_type) { @@ -1702,6 +1709,8 @@ static void vgic_mmio_write_its_baser(struct kvm *kvm, vgic_its_free_collection_list(kvm, its); break; } + /* A concurrent injection may have cached a translation. */ + vgic_its_invalidate_cache(its); mutex_unlock(&its->its_lock); } } @@ -2019,18 +2028,22 @@ static int vgic_its_attr_regs_access(struct kvm_device *dev, return ret; } -static u32 compute_next_devid_offset(struct list_head *h, +static u32 compute_next_devid_offset(struct vgic_its *its, u64 baser, struct its_device *dev) { - struct its_device *next; - u32 next_offset; + struct its_device *next = dev; - if (list_is_last(&dev->dev_list, h)) - return 0; - next = list_next_entry(dev, dev_list); - next_offset = next->device_id - dev->device_id; + /* + * Point at the next device vgic_its_save_device_tables() saves. It + * sorts device_list first, so the subtraction cannot underflow. + */ + list_for_each_entry_continue(next, &its->device_list, dev_list) { + if (vgic_its_check_id(its, baser, next->device_id, NULL)) + return min_t(u32, next->device_id - dev->device_id, + VITS_DTE_MAX_DEVID_OFFSET); + } - return min_t(u32, next_offset, VITS_DTE_MAX_DEVID_OFFSET); + return 0; } static u32 compute_next_eventid_offset(struct list_head *h, struct its_ite *ite) @@ -2271,17 +2284,18 @@ static int vgic_its_restore_itt(struct vgic_its *its, struct its_device *dev) * vgic_its_save_dte - Save a device table entry at a given GPA * * @its: ITS handle + * @baser: GITS_BASER the caller is saving against * @dev: ITS device * @ptr: GPA */ -static int vgic_its_save_dte(struct vgic_its *its, struct its_device *dev, - gpa_t ptr) +static int vgic_its_save_dte(struct vgic_its *its, u64 baser, + struct its_device *dev, gpa_t ptr) { u64 val, itt_addr_field; u32 next_offset; itt_addr_field = dev->itt_addr >> 8; - next_offset = compute_next_devid_offset(&its->device_list, dev); + next_offset = compute_next_devid_offset(its, baser, dev); val = (1ULL << KVM_ITS_DTE_VALID_SHIFT | ((u64)next_offset << KVM_ITS_DTE_NEXT_SHIFT) | (itt_addr_field << KVM_ITS_DTE_ITTADDR_SHIFT) | @@ -2380,15 +2394,16 @@ static int vgic_its_save_device_tables(struct vgic_its *its) int ret; gpa_t eaddr; + /* Don't fail a save that userspace must be able to issue. */ if (!vgic_its_check_id(its, baser, dev->device_id, &eaddr)) - return -EINVAL; + continue; ret = vgic_its_save_itt(its, dev); if (ret) return ret; - ret = vgic_its_save_dte(its, dev, eaddr); + ret = vgic_its_save_dte(its, baser, dev, eaddr); if (ret) return ret; } @@ -2541,9 +2556,6 @@ static int vgic_its_save_collection_table(struct vgic_its *its) max_size = GITS_BASER_NR_PAGES(baser) * SZ_64K; list_for_each_entry(collection, &its->collection_list, coll_list) { - if (!vgic_its_check_id(its, baser, collection->collection_id, NULL)) - return -EINVAL; - ret = vgic_its_save_cte(its, collection, gpa); if (ret) return ret; diff --git a/arch/arm64/kvm/vgic/vgic-v2.c b/arch/arm64/kvm/vgic/vgic-v2.c index 7182f63fc938..70cc53ba810a 100644 --- a/arch/arm64/kvm/vgic/vgic-v2.c +++ b/arch/arm64/kvm/vgic/vgic-v2.c @@ -122,6 +122,10 @@ void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu) for (int lr = 0; lr < vgic_cpu->vgic_v2.used_lrs; lr++) vgic_v2_fold_lr(vcpu, cpuif->vgic_lr[lr]); + cpuif->used_lrs = 0; + if (!irq) + return; + /* See the GICv3 equivalent for the EOIcount handling rationale */ list_for_each_entry_continue(irq, &vgic_cpu->ap_list_head, ap_list) { u32 lr; @@ -144,8 +148,6 @@ void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu) vgic_v2_fold_lr(vcpu, lr); eoicount--; } - - cpuif->used_lrs = 0; } void vgic_v2_deactivate(struct kvm_vcpu *vcpu, u32 val) diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c index 726e20a1da6e..346bacb3198f 100644 --- a/arch/arm64/kvm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -155,6 +155,10 @@ void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu) for (int lr = 0; lr < cpuif->used_lrs; lr++) vgic_v3_fold_lr(vcpu, cpuif->vgic_lr[lr]); + cpuif->used_lrs = 0; + if (!irq) + return; + /* * EOIMode=0: use EOIcount to emulate deactivation. We are * guaranteed to deactivate in reverse order of the activation, so @@ -188,8 +192,6 @@ void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu) vgic_v3_fold_lr(vcpu, lr); eoicount--; } - - cpuif->used_lrs = 0; } void vgic_v3_deactivate(struct kvm_vcpu *vcpu, u64 val) diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index b25303d9919f..5cf5a1ef86cd 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -853,6 +853,17 @@ static void vgic_prune_ap_list(struct kvm_vcpu *vcpu) goto retry; } + /* + * Fix the last_lr_irq refcount which was obtained while + * populating the LRs. This can also result in the LPI being + * deleted. + */ + irq = *host_data_ptr(last_lr_irq); + if (irq) { + deleted_lpis |= vgic_put_irq_norelease(vcpu->kvm, irq); + *host_data_ptr(last_lr_irq) = NULL; + } + raw_spin_unlock(&vgic_cpu->ap_list_lock); if (unlikely(deleted_lpis)) @@ -866,9 +877,6 @@ static void vgic_fold_state(struct kvm_vcpu *vcpu) return; } - if (!*host_data_ptr(last_lr_irq)) - return; - if (kvm_vgic_global_state.type == VGIC_V2) vgic_v2_fold_lr_state(vcpu); else @@ -1021,11 +1029,14 @@ static void vgic_flush_lr_state(struct kvm_vcpu *vcpu) scoped_guard(raw_spinlock, &irq->irq_lock) { if (likely(vgic_target_oracle(irq) == vcpu)) { vgic_populate_lr(vcpu, irq, count++); - *host_data_ptr(last_lr_irq) = irq; + if (count == kvm_vgic_global_state.nr_lr) { + vgic_get_irq_ref(irq); + *host_data_ptr(last_lr_irq) = irq; + } } } - if (count == kvm_vgic_global_state.nr_lr) + if (*host_data_ptr(last_lr_irq)) break; } diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 79d90226fd5d..d4384131e10d 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -2295,7 +2295,7 @@ pte_t modify_prot_start_ptes(struct vm_area_struct *vma, unsigned long addr, * in cases where cpu is affected with errata #2645198. */ if (pte_accessible(vma->vm_mm, pte) && pte_user_exec(pte)) - __flush_tlb_range(vma, addr, nr * PAGE_SIZE, + __flush_tlb_range(vma, addr, addr + nr * PAGE_SIZE, PAGE_SIZE, 3, TLBF_NOWALKCACHE); } diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index c18e005a41db..c1279fabce47 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -600,6 +600,8 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf) * 12 registers are on the stack */ emit(A64_SUB_I(1, A64_SP, A64_FP, 96), ctx); + /* The callback may use its own BPF stack, set up fp for it. */ + ctx->fp_used = true; } /* Stack must be multiples of 16B */ @@ -2416,10 +2418,11 @@ bool bpf_jit_supports_subprog_tailcalls(void) return true; } -static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_node *node, - int bargs_off, int retval_off, int run_ctx_off, - bool save_ret) +static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_image *im, + struct bpf_tramp_node *node, int bargs_off, + int retval_off, int run_ctx_off, bool save_ret) { + void *skip; __le32 *branch; u64 enter_prog; u64 exit_prog; @@ -2429,6 +2432,10 @@ static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_node *node, enter_prog = (u64)bpf_trampoline_enter(p); exit_prog = (u64)bpf_trampoline_exit(p); + /* nop, patched to skip this prog when it is detached */ + skip = ctx->ro_image + ctx->idx; + emit(A64_NOP, ctx); + if (node->cookie == 0) { /* if cookie is zero, one instruction is enough to store it */ emit(A64_STR64I(A64_ZR, A64_SP, run_ctx_off + cookie_off), ctx); @@ -2481,11 +2488,13 @@ static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_node *node, emit(A64_ADD_I(1, A64_R(2), A64_SP, run_ctx_off), ctx); emit_call(exit_prog, ctx); + + bpf_tramp_image_add_skip(im, p, skip, ctx->ro_image + ctx->idx); } -static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_nodes *tn, - int bargs_off, int retval_off, int run_ctx_off, - __le32 **branches) +static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_image *im, + struct bpf_tramp_nodes *tn, int bargs_off, + int retval_off, int run_ctx_off, __le32 **branches) { int i; @@ -2494,7 +2503,7 @@ static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_nodes *tn, */ emit(A64_STR64I(A64_ZR, A64_SP, retval_off), ctx); for (i = 0; i < tn->nr_nodes; i++) { - invoke_bpf_prog(ctx, tn->nodes[i], bargs_off, retval_off, + invoke_bpf_prog(ctx, im, tn->nodes[i], bargs_off, retval_off, run_ctx_off, true); /* if (*(u64 *)(sp + retval_off) != 0) * goto do_fexit; @@ -2880,7 +2889,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, store_func_meta(ctx, meta, func_meta_off); cookie_bargs_off--; } - invoke_bpf_prog(ctx, fentry->nodes[i], bargs_off, + invoke_bpf_prog(ctx, im, fentry->nodes[i], bargs_off, retval_off, run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET); } @@ -2891,7 +2900,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, if (!branches) return -ENOMEM; - invoke_bpf_mod_ret(ctx, fmod_ret, bargs_off, retval_off, + invoke_bpf_mod_ret(ctx, im, fmod_ret, bargs_off, retval_off, run_ctx_off, branches); } @@ -2904,9 +2913,6 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, emit(A64_RET(A64_R(10)), ctx); /* store return value */ emit(A64_STR64I(A64_R(0), A64_SP, retval_off), ctx); - /* reserve a nop for bpf_tramp_image_put */ - im->ip_after_call = ctx->ro_image + ctx->idx; - emit(A64_NOP, ctx); } /* update the branches saved in invoke_bpf_mod_ret with cbnz */ @@ -2928,12 +2934,11 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, store_func_meta(ctx, meta, func_meta_off); cookie_bargs_off--; } - invoke_bpf_prog(ctx, fexit->nodes[i], bargs_off, retval_off, + invoke_bpf_prog(ctx, im, fexit->nodes[i], bargs_off, retval_off, run_ctx_off, false); } if (flags & BPF_TRAMP_F_CALL_ORIG) { - im->ip_epilogue = ctx->ro_image + ctx->idx; /* for the first pass, assume the worst case */ if (!ctx->image) ctx->idx += 4; @@ -2992,7 +2997,7 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, .image = NULL, .idx = 0, }; - struct bpf_tramp_image im; + struct bpf_tramp_image im = {}; struct arg_aux aaux; int ret; @@ -3279,6 +3284,10 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, * longer reachable, since bpf_tramp_image_put() function already * uses percpu_ref and task-based rcu to do the sync, no need to call * the sync version here, see bpf_tramp_image_put() for details. + * + * 3. when a detached prog is patched out of a trampoline, a CPU that + * still executes the old nop calls the prog before it went through + * a quiescent state, and the prog is freed after grace periods. */ ret = aarch64_insn_patch_text_nosync(ip, new_insn); out: diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 2775ba3359cf..58b6c084f9f2 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -105,7 +105,6 @@ WORKAROUND_1902691 WORKAROUND_2038923 WORKAROUND_2064142 WORKAROUND_2077057 -WORKAROUND_2457168 WORKAROUND_2645198 WORKAROUND_2658417 WORKAROUND_4193714 @@ -132,3 +131,4 @@ WORKAROUND_REPEAT_TLBI_SYNC WORKAROUND_SPECULATIVE_AT WORKAROUND_SPECULATIVE_SSBS WORKAROUND_SPECULATIVE_UNPRIV_LOAD +WORKAROUND_BROKEN_AMU_CONSTCNT diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 4da278900938..e78eb582c300 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -1578,6 +1578,24 @@ void *bpf_arch_text_copy(void *dst, void *src, size_t len) return ret ? ERR_PTR(-EINVAL) : dst; } +int arch_bpf_trampoline_skip(void *nop, void *target) +{ + u32 old_insn = INSN_NOP; + u32 new_insn = larch_insn_gen_b((unsigned long)nop, (unsigned long)target); + int ret; + + if (memcmp(nop, &old_insn, LOONGARCH_INSN_SIZE)) + return -EFAULT; + + cpus_read_lock(); + mutex_lock(&text_mutex); + ret = larch_insn_text_copy(nop, &new_insn, LOONGARCH_INSN_SIZE); + mutex_unlock(&text_mutex); + cpus_read_unlock(); + + return ret; +} + int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, enum bpf_text_poke_type new_t, void *old_addr, void *new_addr) @@ -1696,13 +1714,18 @@ static void restore_stk_args(struct jit_ctx *ctx, int nr_stk_args, int args_off, } } -static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_node *n, - int args_off, int retval_off, int run_ctx_off, bool save_ret) +static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_image *im, + struct bpf_tramp_node *n, int args_off, int retval_off, + int run_ctx_off, bool save_ret) { int ret; u32 *branch; struct bpf_prog *p = n->link->prog; int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); + void *skip = ctx->ro_image + ctx->idx; + + /* nop, patched to a b over this prog when it is detached */ + emit_insn(ctx, nop); if (n->cookie) emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, @@ -1755,13 +1778,17 @@ static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_node *n, /* arg3: &run_ctx */ emit_insn(ctx, addid, LOONGARCH_GPR_A2, LOONGARCH_GPR_FP, -run_ctx_off); ret = emit_call(ctx, (const u64)bpf_trampoline_exit(p)); + if (ret) + return ret; - return ret; + bpf_tramp_image_add_skip(im, p, skip, ctx->ro_image + ctx->idx); + return 0; } -static int invoke_bpf(struct jit_ctx *ctx, struct bpf_tramp_nodes *tn, - int args_off, int retval_off, int run_ctx_off, - int func_meta_off, bool save_ret, u64 func_meta, int cookie_off) +static int invoke_bpf(struct jit_ctx *ctx, struct bpf_tramp_image *im, + struct bpf_tramp_nodes *tn, int args_off, int retval_off, + int run_ctx_off, int func_meta_off, bool save_ret, + u64 func_meta, int cookie_off) { int i, cur_cookie = (cookie_off - args_off) / 8; @@ -1774,7 +1801,8 @@ static int invoke_bpf(struct jit_ctx *ctx, struct bpf_tramp_nodes *tn, emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, -func_meta_off, meta); cur_cookie--; } - err = invoke_bpf_prog(ctx, tn->nodes[i], args_off, retval_off, run_ctx_off, save_ret); + err = invoke_bpf_prog(ctx, im, tn->nodes[i], args_off, retval_off, + run_ctx_off, save_ret); if (err) return err; } @@ -2017,7 +2045,7 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i } if (fentry->nr_nodes) { - ret = invoke_bpf(ctx, fentry, args_off, retval_off, run_ctx_off, func_meta_off, + ret = invoke_bpf(ctx, im, fentry, args_off, retval_off, run_ctx_off, func_meta_off, flags & BPF_TRAMP_F_RET_FENTRY_RET, func_meta, cookie_off); if (ret) return ret; @@ -2029,7 +2057,7 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -retval_off); for (i = 0; i < fmod_ret->nr_nodes; i++) { - ret = invoke_bpf_prog(ctx, fmod_ret->nodes[i], + ret = invoke_bpf_prog(ctx, im, fmod_ret->nodes[i], args_off, retval_off, run_ctx_off, true); if (ret) goto out; @@ -2051,10 +2079,6 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i goto out; emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off); emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8)); - im->ip_after_call = ctx->ro_image + ctx->idx; - /* Reserve space for the move_imm + jirl instruction */ - for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++) - emit_insn(ctx, nop); } for (i = 0; ctx->image && i < fmod_ret->nr_nodes; i++) { @@ -2068,14 +2092,13 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, -func_meta_off, func_meta); if (fexit->nr_nodes) { - ret = invoke_bpf(ctx, fexit, args_off, retval_off, run_ctx_off, + ret = invoke_bpf(ctx, im, fexit, args_off, retval_off, run_ctx_off, func_meta_off, false, func_meta, cookie_off); if (ret) goto out; } if (flags & BPF_TRAMP_F_CALL_ORIG) { - im->ip_epilogue = ctx->ro_image + ctx->idx; move_addr(ctx, LOONGARCH_GPR_A0, (const u64)im); ret = emit_call(ctx, (const u64)__bpf_tramp_exit); if (ret) @@ -2177,11 +2200,8 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, struct bpf_tramp_nodes *tnodes, void *func_addr) { int ret; - struct jit_ctx ctx; - struct bpf_tramp_image im; - - ctx.image = NULL; - ctx.idx = 0; + struct jit_ctx ctx = {}; + struct bpf_tramp_image im = {}; ret = __arch_prepare_bpf_trampoline(&ctx, &im, m, tnodes, func_addr, flags); diff --git a/arch/loongarch/vdso/Makefile b/arch/loongarch/vdso/Makefile index 9c9181bb4071..0b84892d084b 100644 --- a/arch/loongarch/vdso/Makefile +++ b/arch/loongarch/vdso/Makefile @@ -16,6 +16,7 @@ ccflags-vdso := \ $(filter -m64,$(KBUILD_CFLAGS)) \ $(filter -march=%,$(KBUILD_CFLAGS)) \ $(filter -m%-float,$(KBUILD_CFLAGS)) \ + $(CONFIG_CC_OPT_INLINE_MEMSET) \ $(CLANG_FLAGS) \ -D__VDSO__ diff --git a/arch/mips/net/bpf_jit_comp32.c b/arch/mips/net/bpf_jit_comp32.c index 40a878b672f5..15a2a153dc87 100644 --- a/arch/mips/net/bpf_jit_comp32.c +++ b/arch/mips/net/bpf_jit_comp32.c @@ -1111,7 +1111,7 @@ static void emit_jmp_i64(struct jit_context *ctx, emit(ctx, xor, tmp, lo(dst), tmp); } if (imm < 0) { /* Compare sign extension */ - emit(ctx, addu, MIPS_R_T9, hi(dst), 1); + emit(ctx, addiu, MIPS_R_T9, hi(dst), 1); emit(ctx, or, tmp, tmp, MIPS_R_T9); } else { /* Compare zero extension */ emit(ctx, or, tmp, tmp, hi(dst)); diff --git a/arch/mips/net/bpf_jit_comp64.c b/arch/mips/net/bpf_jit_comp64.c index fa7e9aa37f49..6681ccac9dd9 100644 --- a/arch/mips/net/bpf_jit_comp64.c +++ b/arch/mips/net/bpf_jit_comp64.c @@ -305,8 +305,7 @@ static void emit_bswap_r64(struct jit_context *ctx, u8 dst, u32 width) case 16: emit_sext(ctx, dst, dst); emit_bswap_r(ctx, dst, width); - if (cpu_has_mips64r2 || cpu_has_mips64r6) - emit_zext(ctx, dst); + emit_zext(ctx, dst); break; } clobber_reg(ctx, dst); diff --git a/arch/parisc/include/asm/compat_ucontext.h b/arch/parisc/include/asm/compat_ucontext.h deleted file mode 100644 index c606f1bc891d..000000000000 --- a/arch/parisc/include/asm/compat_ucontext.h +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_PARISC_COMPAT_UCONTEXT_H -#define _ASM_PARISC_COMPAT_UCONTEXT_H - -#include - -/* 32-bit ucontext as seen from an 64-bit kernel */ -struct compat_ucontext { - compat_uint_t uc_flags; - compat_uptr_t uc_link; - compat_stack_t uc_stack; /* struct compat_sigaltstack (12 bytes)*/ - /* FIXME: Pad out to get uc_mcontext to start at an 8-byte aligned boundary */ - compat_uint_t pad[1]; - struct compat_sigcontext uc_mcontext; - compat_sigset_t uc_sigmask; /* mask last for extensibility */ -}; - -#endif /* !_ASM_PARISC_COMPAT_UCONTEXT_H */ diff --git a/arch/parisc/include/asm/thread_info.h b/arch/parisc/include/asm/thread_info.h index b283738bb6da..249370706242 100644 --- a/arch/parisc/include/asm/thread_info.h +++ b/arch/parisc/include/asm/thread_info.h @@ -24,7 +24,7 @@ struct thread_info { /* thread information allocation */ -#ifdef CONFIG_IRQSTACKS +#if defined(CONFIG_IRQSTACKS) && !defined(CONFIG_64BIT) #define THREAD_SIZE_ORDER 2 /* PA-RISC requires at least 16k stack */ #else #define THREAD_SIZE_ORDER 3 /* PA-RISC requires at least 32k stack */ diff --git a/arch/parisc/kernel/setup.c b/arch/parisc/kernel/setup.c index d3e17a7a8901..4d3015a411d6 100644 --- a/arch/parisc/kernel/setup.c +++ b/arch/parisc/kernel/setup.c @@ -132,6 +132,18 @@ void __init setup_arch(char **cmdline_p) parisc_cache_init(); paging_init(); + /* + * Parse early parameters before mm_core_init_early() runs. + * Several early_param() handlers only record data that is consumed + * from there - for example hugepages=, hugepagesz=, + * default_hugepagesz=, hugetlb_cma= and hugetlb_free_vmemmap= - so + * the generic parse_early_param() call in start_kernel() is too late + * for them. jump_label_init() must come first, since early param + * handlers may enable or disable static keys. + */ + jump_label_init(); + parse_early_param(); + #ifdef CONFIG_PA11 dma_ops_init(); #endif diff --git a/arch/parisc/kernel/unwind.c b/arch/parisc/kernel/unwind.c index 32103a270a8e..fab9ae22191a 100644 --- a/arch/parisc/kernel/unwind.c +++ b/arch/parisc/kernel/unwind.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -49,27 +50,23 @@ static DEFINE_SPINLOCK(unwind_lock); static struct unwind_table kernel_unwind_table __ro_after_init; static LIST_HEAD(unwind_tables); +static int cmp_unwind_entry(const void *key, const void *elt) +{ + unsigned long addr = (unsigned long)key; + const struct unwind_table_entry *e = elt; + + if (addr < e->region_start) + return -1; + if (addr > e->region_end) + return 1; + return 0; +} + static inline const struct unwind_table_entry * find_unwind_entry_in_table(const struct unwind_table *table, unsigned long addr) { - const struct unwind_table_entry *e = NULL; - unsigned long lo, hi, mid; - - lo = 0; - hi = table->length - 1; - - while (lo <= hi) { - mid = (hi - lo) / 2 + lo; - e = &table->table[mid]; - if (addr < e->region_start) - hi = mid - 1; - else if (addr > e->region_end) - lo = mid + 1; - else - return e; - } - - return NULL; + return bsearch((void *)addr, table->table, table->length, + sizeof(*table->table), cmp_unwind_entry); } static const struct unwind_table_entry * diff --git a/arch/powerpc/kernel/vdso/Makefile b/arch/powerpc/kernel/vdso/Makefile index 368759f81708..0d1a49529885 100644 --- a/arch/powerpc/kernel/vdso/Makefile +++ b/arch/powerpc/kernel/vdso/Makefile @@ -43,6 +43,7 @@ ccflags-y := -fno-common -fno-builtin -DBUILD_VDSO ccflags-y += $(DISABLE_LATENT_ENTROPY_PLUGIN) ccflags-y += $(call cc-option, -fno-stack-protector) ccflags-y += -DDISABLE_BRANCH_PROFILING +ccflags-y += $(CONFIG_CC_OPT_INLINE_MEMSET) ccflags-y += -ffreestanding -fasynchronous-unwind-tables ccflags-remove-y := $(CC_FLAGS_FTRACE) ldflags-y := -Wl,--hash-style=both -nostdlib -shared -z noexecstack $(CLANG_FLAGS) diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index dbac3573b2c8..de05d721edc4 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -3058,7 +3058,6 @@ static int kvmppc_core_vcpu_create_hv(struct kvm_vcpu *vcpu) init_waitqueue_head(&vcpu->arch.cpu_run); - mutex_lock(&kvm->lock); vcore = NULL; err = -EINVAL; if (cpu_has_feature(CPU_FTR_ARCH_300)) { @@ -3091,7 +3090,6 @@ static int kvmppc_core_vcpu_create_hv(struct kvm_vcpu *vcpu) mutex_unlock(&kvm->arch.mmu_setup_lock); } } - mutex_unlock(&kvm->lock); if (!vcore) return err; diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 7b07b43575f1..7a612688e7a2 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -602,14 +602,18 @@ int arch_protect_bpf_trampoline(void *image, unsigned int size) } static int invoke_bpf_prog(u32 *image, u32 *ro_image, struct codegen_context *ctx, - struct bpf_tramp_node *n, int regs_off, int retval_off, - int run_ctx_off, bool save_ret) + struct bpf_tramp_image *im, struct bpf_tramp_node *n, + int regs_off, int retval_off, int run_ctx_off, bool save_ret) { struct bpf_prog *p = n->link->prog; ppc_inst_t branch_insn; - u32 jmp_idx; + u32 jmp_idx, skip_idx; int ret = 0; + /* nop, patched to skip this prog when it is detached */ + skip_idx = ctx->idx; + EMIT(PPC_RAW_NOP()); + /* Save cookie */ if (IS_ENABLED(CONFIG_PPC64)) { PPC_LI64(_R3, n->cookie); @@ -679,13 +683,17 @@ static int invoke_bpf_prog(u32 *image, u32 *ro_image, struct codegen_context *ct EMIT(PPC_RAW_ADDI(_R5, _R1, run_ctx_off)); ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx, (unsigned long)bpf_trampoline_exit(p)); + if (ret) + return ret; - return ret; + if (ro_image) /* image is NULL for dummy pass */ + bpf_tramp_image_add_skip(im, p, &ro_image[skip_idx], &ro_image[ctx->idx]); + return 0; } static int invoke_bpf_mod_ret(u32 *image, u32 *ro_image, struct codegen_context *ctx, - struct bpf_tramp_nodes *tn, int regs_off, int retval_off, - int run_ctx_off, u32 *branches) + struct bpf_tramp_image *im, struct bpf_tramp_nodes *tn, + int regs_off, int retval_off, int run_ctx_off, u32 *branches) { int i; @@ -696,8 +704,8 @@ static int invoke_bpf_mod_ret(u32 *image, u32 *ro_image, struct codegen_context EMIT(PPC_RAW_LI(_R3, 0)); EMIT(PPC_RAW_STL(_R3, _R1, retval_off)); for (i = 0; i < tn->nr_nodes; i++) { - if (invoke_bpf_prog(image, ro_image, ctx, tn->nodes[i], regs_off, retval_off, - run_ctx_off, true)) + if (invoke_bpf_prog(image, ro_image, ctx, im, tn->nodes[i], regs_off, + retval_off, run_ctx_off, true)) return -EINVAL; /* @@ -1043,8 +1051,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im cookie_ctx_off--; } - if (invoke_bpf_prog(image, ro_image, ctx, fentry->nodes[i], regs_off, retval_off, - run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET)) + if (invoke_bpf_prog(image, ro_image, ctx, im, fentry->nodes[i], regs_off, + retval_off, run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET)) return -EINVAL; } @@ -1053,7 +1061,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im if (!branches) return -ENOMEM; - if (invoke_bpf_mod_ret(image, ro_image, ctx, fmod_ret, regs_off, retval_off, + if (invoke_bpf_mod_ret(image, ro_image, ctx, im, fmod_ret, regs_off, retval_off, run_ctx_off, branches)) { ret = -EINVAL; goto cleanup; @@ -1090,11 +1098,6 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im /* Restore updated tail_call_cnt */ if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) bpf_trampoline_restore_tail_call_cnt(image, ctx, bpf_frame_size, r4_off); - - /* Reserve space to patch branch instruction to skip fexit progs */ - if (ro_image) /* image is NULL for dummy pass */ - im->ip_after_call = &((u32 *)ro_image)[ctx->idx]; - EMIT(PPC_RAW_NOP()); } /* Update branches saved in invoke_bpf_mod_ret with address of do_fexit */ @@ -1123,16 +1126,14 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im cookie_ctx_off--; } - if (invoke_bpf_prog(image, ro_image, ctx, fexit->nodes[i], regs_off, retval_off, - run_ctx_off, false)) { + if (invoke_bpf_prog(image, ro_image, ctx, im, fexit->nodes[i], regs_off, + retval_off, run_ctx_off, false)) { ret = -EINVAL; goto cleanup; } } if (flags & BPF_TRAMP_F_CALL_ORIG) { - if (ro_image) /* image is NULL for dummy pass */ - im->ip_epilogue = &((u32 *)ro_image)[ctx->idx]; PPC_LI_ADDR(_R3, im); ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx, (unsigned long)__bpf_tramp_exit); @@ -1192,7 +1193,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, struct bpf_tramp_nodes *tnodes, void *func_addr) { - struct bpf_tramp_image im; + struct bpf_tramp_image im = {}; int ret; ret = __arch_prepare_bpf_trampoline(&im, NULL, NULL, NULL, m, flags, tnodes, func_addr); @@ -1320,7 +1321,7 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, /* * If we are not poking at bpf prog entry, then we are simply patching in/out - * an unconditional branch instruction at im->ip_after_call + * an unconditional branch instruction in a trampoline image */ if (offset) { if (old_t == BPF_MOD_CALL || new_t == BPF_MOD_CALL) { diff --git a/arch/powerpc/perf/imc-pmu.c b/arch/powerpc/perf/imc-pmu.c index f401181d8669..a9f8464997a2 100644 --- a/arch/powerpc/perf/imc-pmu.c +++ b/arch/powerpc/perf/imc-pmu.c @@ -1275,6 +1275,8 @@ static int trace_imc_prepare_sample(struct trace_imc_data *mem, struct perf_event_header *header, struct perf_event *event) { + u16 misc = 0; + /* Sanity checks for a valid record */ if (be64_to_cpu(READ_ONCE(mem->tb1)) > *prev_tb) *prev_tb = be64_to_cpu(READ_ONCE(mem->tb1)); @@ -1289,23 +1291,19 @@ static int trace_imc_prepare_sample(struct trace_imc_data *mem, data->ip = be64_to_cpu(READ_ONCE(mem->ip)); data->period = event->hw.last_period; - header->type = PERF_RECORD_SAMPLE; - header->size = sizeof(*header) + event->header_size; - header->misc = 0; - if (cpu_has_feature(CPU_FTR_ARCH_31)) { switch (IMC_TRACE_RECORD_VAL_HVPR(be64_to_cpu(READ_ONCE(mem->val)))) { case 0:/* when MSR HV and PR not set in the trace-record */ - header->misc |= PERF_RECORD_MISC_GUEST_KERNEL; + misc |= PERF_RECORD_MISC_GUEST_KERNEL; break; case 1: /* MSR HV is 0 and PR is 1 */ - header->misc |= PERF_RECORD_MISC_GUEST_USER; + misc |= PERF_RECORD_MISC_GUEST_USER; break; case 2: /* MSR HV is 1 and PR is 0 */ - header->misc |= PERF_RECORD_MISC_KERNEL; + misc |= PERF_RECORD_MISC_KERNEL; break; case 3: /* MSR HV is 1 and PR is 1 */ - header->misc |= PERF_RECORD_MISC_USER; + misc |= PERF_RECORD_MISC_USER; break; default: pr_info("IMC: Unable to set the flag based on MSR bits\n"); @@ -1313,11 +1311,14 @@ static int trace_imc_prepare_sample(struct trace_imc_data *mem, } } else { if (is_kernel_addr(data->ip)) - header->misc |= PERF_RECORD_MISC_KERNEL; + misc |= PERF_RECORD_MISC_KERNEL; else - header->misc |= PERF_RECORD_MISC_USER; + misc |= PERF_RECORD_MISC_USER; } - perf_event_header__init_id(header, data, event); + perf_event_header__init(header, data, + PERF_RECORD_SAMPLE, misc, + sizeof(*header) + event->header_size, + event); return 0; } diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h index 6c823361be86..621f84d1898f 100644 --- a/arch/riscv/include/asm/csr.h +++ b/arch/riscv/include/asm/csr.h @@ -188,12 +188,24 @@ #define HGATP_MODE_SHIFT HGATP32_MODE_SHIFT #endif -/* VSIP & HVIP relation */ +/* + * VSIP & HVIP relation + * + * The bit positions are same between VSIP and HVIP for interrupt + * numbers 13-63, where there's a shift for the SSI, STI and SEI. + */ #define VSIP_TO_HVIP_SHIFT (IRQ_VS_SOFT - IRQ_S_SOFT) -#define VSIP_VALID_MASK ((_AC(1, UL) << IRQ_S_SOFT) | \ +#define VSIP_BIAS_MASK ((_AC(1, UL) << IRQ_S_SOFT) | \ (_AC(1, UL) << IRQ_S_TIMER) | \ - (_AC(1, UL) << IRQ_S_EXT) | \ - (_AC(1, UL) << IRQ_PMU_OVF)) + (_AC(1, UL) << IRQ_S_EXT)) +#define VSIP_NO_BIAS_MASK (_AC(1, UL) << IRQ_PMU_OVF) +#define VSIP_VALID_MASK (VSIP_BIAS_MASK | VSIP_NO_BIAS_MASK) +#define vsip_to_hvip(_vsip) ((((_vsip) & VSIP_BIAS_MASK) << \ + VSIP_TO_HVIP_SHIFT) | \ + ((_vsip) & VSIP_NO_BIAS_MASK)) +#define hvip_to_vsip(_hvip) ((((_hvip) >> VSIP_TO_HVIP_SHIFT) & \ + VSIP_BIAS_MASK) | \ + ((_hvip) & VSIP_NO_BIAS_MASK)) /* AIA CSR bits */ #define TOPI_IID_SHIFT 16 diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile index 8dbf2532a573..c023046a3fd7 100644 --- a/arch/riscv/kernel/vdso/Makefile +++ b/arch/riscv/kernel/vdso/Makefile @@ -36,6 +36,7 @@ endif ccflags-y := -fno-stack-protector ccflags-y += -DDISABLE_BRANCH_PROFILING ccflags-y += -fno-builtin +ccflags-y += $(CONFIG_CC_OPT_INLINE_MEMSET) ccflags-y += $(KBUILD_BASE_ISA)$(CFI_MARCH) ccflags-y += $(CFI_FULL) asflags-y += $(KBUILD_BASE_ISA)$(CFI_MARCH) diff --git a/arch/riscv/kvm/aia_device.c b/arch/riscv/kvm/aia_device.c index efc7c0bcfba9..97833a04268a 100644 --- a/arch/riscv/kvm/aia_device.c +++ b/arch/riscv/kvm/aia_device.c @@ -237,7 +237,7 @@ static int aia_init(struct kvm *kvm) return -EBUSY; /* We might be in the middle of creating a VCPU? */ - if (kvm->created_vcpus != atomic_read(&kvm->online_vcpus)) + if (kvm_is_vcpu_creation_in_progress(kvm)) return -EBUSY; /* Number of sources should be less than or equals number of IDs */ diff --git a/arch/riscv/kvm/aia_imsic.c b/arch/riscv/kvm/aia_imsic.c index c1af23e79ae0..f17dc559a709 100644 --- a/arch/riscv/kvm/aia_imsic.c +++ b/arch/riscv/kvm/aia_imsic.c @@ -965,9 +965,14 @@ int kvm_riscv_aia_imsic_rw_attr(struct kvm *kvm, unsigned long type, if (!vcpu) return -ENODEV; + if (mutex_lock_killable(&vcpu->mutex)) + return -EINTR; + imsic = vcpu->arch.aia_context.imsic_state; - if (!imsic) - return -ENODEV; + if (!imsic) { + rc = -ENODEV; + goto out_unlock; + } isel = KVM_DEV_RISCV_AIA_IMSIC_GET_ISEL(type); read_lock_irqsave(&imsic->vsfile_lock, flags); @@ -991,6 +996,8 @@ int kvm_riscv_aia_imsic_rw_attr(struct kvm *kvm, unsigned long type, rc = imsic_vsfile_rw(vsfile_hgei, vsfile_cpu, imsic->nr_eix, isel, write, val); +out_unlock: + mutex_unlock(&vcpu->mutex); return rc; } diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c index 6035b5ec9503..3e955d808743 100644 --- a/arch/riscv/kvm/mmu.c +++ b/arch/riscv/kvm/mmu.c @@ -627,6 +627,7 @@ int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, int ret; kvm_pfn_t hfn; bool is_hugetlb; + bool unused = false; bool writable; unsigned int vma_pageshift; gfn_t gfn = gpa >> PAGE_SHIFT; @@ -707,6 +708,8 @@ int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, vma_pageshift, current); return 0; } + if (is_sigpending_pfn(hfn)) + return -EINTR; if (is_error_noslot_pfn(hfn)) return -EFAULT; @@ -719,8 +722,10 @@ int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, write_lock(&kvm->mmu_lock); - if (mmu_invalidate_retry(kvm, mmu_seq)) + if (mmu_invalidate_retry(kvm, mmu_seq)) { + unused = true; goto out_unlock; + } /* * Check if we are backed by a THP and thus use block mapping if @@ -743,7 +748,8 @@ int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, kvm_err("Failed to map in G-stage\n"); out_unlock: - kvm_release_faultin_page(kvm, page, ret && ret != -EEXIST, writable); + kvm_release_faultin_page(kvm, page, + unused || (ret && ret != -EEXIST), writable); write_unlock(&kvm->mmu_lock); return ret; } diff --git a/arch/riscv/kvm/nacl.c b/arch/riscv/kvm/nacl.c index 9aff03c4f667..a5cda9a65156 100644 --- a/arch/riscv/kvm/nacl.c +++ b/arch/riscv/kvm/nacl.c @@ -42,12 +42,24 @@ void __kvm_riscv_nacl_hfence(void *shmem, } } - entp = shmem + SBI_NACL_SHMEM_HFENCE_ENTRY_CONFIG(i); - *entp = cpu_to_lelong(control); + /* + * Per SBI v3.0 section 15.1.2, the Page_Number and Page_Count + * words must be updated before the Config word with its Pending + * bit set. WRITE_ONCE() stops the compiler from reordering the + * stores and smp_wmb() makes the parameter words globally + * visible to the SBI implementation (or NACL hardware) before + * the Pending bit is set. + */ entp = shmem + SBI_NACL_SHMEM_HFENCE_ENTRY_PNUM(i); - *entp = cpu_to_lelong(page_num); + WRITE_ONCE(*entp, cpu_to_lelong(page_num)); entp = shmem + SBI_NACL_SHMEM_HFENCE_ENTRY_PCOUNT(i); - *entp = cpu_to_lelong(page_count); + WRITE_ONCE(*entp, cpu_to_lelong(page_count)); + + /* Ensure the parameter words are visible before the Pending bit */ + smp_wmb(); + + entp = shmem + SBI_NACL_SHMEM_HFENCE_ENTRY_CONFIG(i); + WRITE_ONCE(*entp, cpu_to_lelong(control)); } int kvm_riscv_nacl_enable(void) diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index e062ca19f9d8..7f1636c44d41 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -491,8 +491,7 @@ bool kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) bool ret; raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); - ie = ((vcpu->arch.guest_csr.vsie & VSIP_VALID_MASK) - << VSIP_TO_HVIP_SHIFT) & (unsigned long)mask; + ie = vsip_to_hvip(vcpu->arch.guest_csr.vsie) & (unsigned long)mask; ie |= vcpu->arch.guest_csr.vsie & ~IRQ_LOCAL_MASK & (unsigned long)mask; ret = vcpu->arch.irqs_pending[0] & ie; diff --git a/arch/riscv/kvm/vcpu_exit.c b/arch/riscv/kvm/vcpu_exit.c index 88e0c369b354..8d36eb8abce1 100644 --- a/arch/riscv/kvm/vcpu_exit.c +++ b/arch/riscv/kvm/vcpu_exit.c @@ -283,7 +283,7 @@ int kvm_riscv_vcpu_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, } /* Print details in-case of error */ - if (ret < 0) { + if (ret < 0 && ret != -EINTR) { kvm_err("VCPU exit error %d\n", ret); kvm_err("SEPC=0x%lx SSTATUS=0x%lx HSTATUS=0x%lx\n", vcpu->arch.guest_context.sepc, diff --git a/arch/riscv/kvm/vcpu_onereg.c b/arch/riscv/kvm/vcpu_onereg.c index 841f2cf87168..f16e25098af1 100644 --- a/arch/riscv/kvm/vcpu_onereg.c +++ b/arch/riscv/kvm/vcpu_onereg.c @@ -272,7 +272,7 @@ static int kvm_riscv_vcpu_general_get_csr(struct kvm_vcpu *vcpu, if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { kvm_riscv_vcpu_flush_interrupts(vcpu); - *out_val = (csr->hvip >> VSIP_TO_HVIP_SHIFT) & VSIP_VALID_MASK; + *out_val = hvip_to_vsip(csr->hvip); *out_val |= csr->hvip & ~IRQ_LOCAL_MASK; } else *out_val = ((unsigned long *)csr)[reg_num]; @@ -293,10 +293,8 @@ static int kvm_riscv_vcpu_general_set_csr(struct kvm_vcpu *vcpu, reg_num = array_index_nospec(reg_num, regs_max); - if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { - reg_val &= VSIP_VALID_MASK; - reg_val <<= VSIP_TO_HVIP_SHIFT; - } + if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) + reg_val = vsip_to_hvip(reg_val); ((unsigned long *)csr)[reg_num] = reg_val; diff --git a/arch/riscv/kvm/vcpu_pmu.c b/arch/riscv/kvm/vcpu_pmu.c index 6ff741ee7803..e095d1ff439f 100644 --- a/arch/riscv/kvm/vcpu_pmu.c +++ b/arch/riscv/kvm/vcpu_pmu.c @@ -270,12 +270,13 @@ static int pmu_ctr_read(struct kvm_vcpu *vcpu, unsigned long cidx, return -EINVAL; pmc->counter_val = kvpmu->fw_event[fevent_code].value; + *out_val = pmc->counter_val; } else if (pmc->perf_event) { - pmc->counter_val += perf_event_read_value(pmc->perf_event, &enabled, &running); + *out_val = pmc->counter_val + + perf_event_read_value(pmc->perf_event, &enabled, &running); } else { return -EINVAL; } - *out_val = pmc->counter_val; return 0; } @@ -454,6 +455,14 @@ int kvm_riscv_vcpu_pmu_snapshot_set_shmem(struct kvm_vcpu *vcpu, unsigned long s } } + /* + * Clear any previously installed snapshot area to avoid leaking + * the old sdata and to keep sdata/snapshot_addr consistent if + * the re-install fails below. + */ + if (kvpmu->snapshot_addr != INVALID_GPA) + kvm_pmu_clear_snapshot_area(vcpu); + kvpmu->sdata = kzalloc(snapshot_area_size, GFP_ATOMIC | __GFP_ACCOUNT); if (!kvpmu->sdata) { sbiret = SBI_ERR_FAILURE; @@ -645,7 +654,6 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base, { struct kvm_pmu *kvpmu = vcpu_to_pmu(vcpu); int i, pmc_index, sbiret = 0; - u64 enabled, running; struct kvm_pmc *pmc; int fevent_code; bool snap_flag_set = flags & SBI_PMU_STOP_FLAG_TAKE_SNAPSHOT; @@ -675,14 +683,19 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base, goto out; } - if (!kvpmu->fw_event[fevent_code].started) + if (!kvpmu->fw_event[fevent_code].started) { sbiret = SBI_ERR_ALREADY_STOPPED; - - kvpmu->fw_event[fevent_code].started = false; + } else { + kvpmu->fw_event[fevent_code].started = false; + pmc->counter_val = kvpmu->fw_event[fevent_code].value; + } } else if (pmc->perf_event) { if (pmc->started) { - /* Stop counting the counter */ - perf_event_disable(pmc->perf_event); + /* + * Stop the counter and fold the live count into counter_val. + * Reset the event value to avoid redundant accumulation. + */ + pmc->counter_val += perf_event_pause(pmc->perf_event, true); pmc->started = false; } else { sbiret = SBI_ERR_ALREADY_STOPPED; @@ -696,11 +709,6 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base, } if (snap_flag_set && !sbiret) { - if (pmc->cinfo.type == SBI_PMU_CTR_TYPE_FW) - pmc->counter_val = kvpmu->fw_event[fevent_code].value; - else if (pmc->perf_event) - pmc->counter_val += perf_event_read_value(pmc->perf_event, - &enabled, &running); /* * The counter and overflow indices in the snapshot region are w.r.to * cbase. Modify the set bit in the counter mask instead of the pmc_index @@ -727,9 +735,10 @@ int kvm_riscv_vcpu_pmu_ctr_stop(struct kvm_vcpu *vcpu, unsigned long ctr_base, } } - if (shmem_needs_update) - kvm_vcpu_write_guest(vcpu, kvpmu->snapshot_addr, kvpmu->sdata, - sizeof(struct riscv_pmu_snapshot_data)); + if (shmem_needs_update && + kvm_vcpu_write_guest(vcpu, kvpmu->snapshot_addr, kvpmu->sdata, + sizeof(struct riscv_pmu_snapshot_data))) + sbiret = SBI_ERR_FAILURE; out: retdata->err_val = sbiret; diff --git a/arch/riscv/kvm/vcpu_sbi_hsm.c b/arch/riscv/kvm/vcpu_sbi_hsm.c index f26207f84bab..06a15629c26b 100644 --- a/arch/riscv/kvm/vcpu_sbi_hsm.c +++ b/arch/riscv/kvm/vcpu_sbi_hsm.c @@ -95,9 +95,9 @@ static int kvm_sbi_ext_hsm_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, ret = kvm_sbi_hsm_vcpu_get_status(vcpu); if (ret >= 0) { retdata->out_val = ret; - retdata->err_val = 0; + ret = 0; } - return 0; + break; case SBI_EXT_HSM_HART_SUSPEND: switch (lower_32_bits(cp->a0)) { case SBI_HSM_SUSPEND_RET_DEFAULT: diff --git a/arch/riscv/kvm/vcpu_timer.c b/arch/riscv/kvm/vcpu_timer.c index ae53133c7ab0..a2cd277a4059 100644 --- a/arch/riscv/kvm/vcpu_timer.c +++ b/arch/riscv/kvm/vcpu_timer.c @@ -61,10 +61,13 @@ static enum hrtimer_restart kvm_riscv_vcpu_hrtimer_expired(struct hrtimer *h) static int kvm_riscv_vcpu_timer_cancel(struct kvm_vcpu_timer *t) { - if (!t->init_done || !t->next_set) + if (!t->init_done) return -EINVAL; hrtimer_cancel(&t->hrt); + + if (!t->next_set) + return -EINVAL; t->next_set = false; return 0; diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 151031e97a24..01fe66774f02 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -822,6 +822,24 @@ static int gen_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call) return emit_jump_and_link(is_call ? RV_REG_T0 : RV_REG_ZERO, rvoff, false, &ctx); } +int arch_bpf_trampoline_skip(void *nop, void *target) +{ + u32 old_insn = rv_nop(); + u32 new_insn = rv_jal(RV_REG_ZERO, (target - nop) >> 1); + int ret; + + if (memcmp(nop, &old_insn, sizeof(old_insn))) + return -EFAULT; + + cpus_read_lock(); + mutex_lock(&text_mutex); + ret = patch_text(nop, &new_insn, sizeof(new_insn)); + mutex_unlock(&text_mutex); + cpus_read_unlock(); + + return ret; +} + int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, enum bpf_text_poke_type new_t, void *old_addr, void *new_addr) @@ -904,12 +922,17 @@ static void emit_store_stack_imm64(u8 reg, int stack_off, u64 imm64, emit_sd(RV_REG_FP, stack_off, reg, ctx); } -static int invoke_bpf_prog(struct bpf_tramp_node *node, int args_off, int retval_off, - int run_ctx_off, bool save_ret, struct rv_jit_context *ctx) +static int invoke_bpf_prog(struct bpf_tramp_image *im, struct bpf_tramp_node *node, + int args_off, int retval_off, int run_ctx_off, bool save_ret, + struct rv_jit_context *ctx) { int ret, branch_off; struct bpf_prog *p = node->link->prog; int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); + void *skip = ctx->ro_insns + ctx->ninsns; + + /* nop, patched to a jal over this prog when it is detached */ + emit(rv_nop(), ctx); if (node->cookie) emit_store_stack_imm64(RV_REG_T1, -run_ctx_off + cookie_off, node->cookie, ctx); @@ -962,13 +985,17 @@ static int invoke_bpf_prog(struct bpf_tramp_node *node, int args_off, int retval /* arg3: &run_ctx */ emit_addi(RV_REG_A2, RV_REG_FP, -run_ctx_off, ctx); ret = emit_call((const u64)bpf_trampoline_exit(p), true, ctx); + if (ret) + return ret; - return ret; + bpf_tramp_image_add_skip(im, p, skip, ctx->ro_insns + ctx->ninsns); + return 0; } -static int invoke_bpf(struct bpf_tramp_nodes *tn, int args_off, int retval_off, - int run_ctx_off, int func_meta_off, bool save_ret, u64 func_meta, - int cookie_off, struct rv_jit_context *ctx) +static int invoke_bpf(struct bpf_tramp_image *im, struct bpf_tramp_nodes *tn, + int args_off, int retval_off, int run_ctx_off, int func_meta_off, + bool save_ret, u64 func_meta, int cookie_off, + struct rv_jit_context *ctx) { int i, cur_cookie = (cookie_off - args_off) / 8; @@ -981,8 +1008,8 @@ static int invoke_bpf(struct bpf_tramp_nodes *tn, int args_off, int retval_off, emit_store_stack_imm64(RV_REG_T1, -func_meta_off, meta, ctx); cur_cookie--; } - err = invoke_bpf_prog(tn->nodes[i], args_off, retval_off, run_ctx_off, - save_ret, ctx); + err = invoke_bpf_prog(im, tn->nodes[i], args_off, retval_off, + run_ctx_off, save_ret, ctx); if (err) return err; } @@ -1170,7 +1197,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, } if (fentry->nr_nodes) { - ret = invoke_bpf(fentry, args_off, retval_off, run_ctx_off, func_meta_off, + ret = invoke_bpf(im, fentry, args_off, retval_off, run_ctx_off, func_meta_off, flags & BPF_TRAMP_F_RET_FENTRY_RET, func_meta, cookie_off, ctx); if (ret) return ret; @@ -1184,7 +1211,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, /* cleanup to avoid garbage return value confusion */ emit_sd(RV_REG_FP, -retval_off, RV_REG_ZERO, ctx); for (i = 0; i < fmod_ret->nr_nodes; i++) { - ret = invoke_bpf_prog(fmod_ret->nodes[i], args_off, retval_off, + ret = invoke_bpf_prog(im, fmod_ret->nodes[i], args_off, retval_off, run_ctx_off, true, ctx); if (ret) goto out; @@ -1211,10 +1238,6 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, emit_sd(RV_REG_FP, -tcc_off, RV_REG_TCC, ctx); emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx); emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx); - im->ip_after_call = ctx->ro_insns + ctx->ninsns; - /* 2 nops reserved for auipc+jalr pair */ - emit(rv_nop(), ctx); - emit(rv_nop(), ctx); } /* update branches saved in invoke_bpf_mod_ret with bnez */ @@ -1230,14 +1253,13 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, emit_store_stack_imm64(RV_REG_T1, -func_meta_off, func_meta, ctx); if (fexit->nr_nodes) { - ret = invoke_bpf(fexit, args_off, retval_off, run_ctx_off, func_meta_off, + ret = invoke_bpf(im, fexit, args_off, retval_off, run_ctx_off, func_meta_off, false, func_meta, cookie_off, ctx); if (ret) goto out; } if (flags & BPF_TRAMP_F_CALL_ORIG) { - im->ip_epilogue = ctx->ro_insns + ctx->ninsns; emit_imm(RV_REG_A0, ctx->insns ? (const s64)im : RV_MAX_COUNT_IMM, ctx); ret = emit_call((const u64)__bpf_tramp_exit, true, ctx); if (ret) @@ -1299,7 +1321,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, struct bpf_tramp_nodes *tnodes, void *func_addr) { - struct bpf_tramp_image im; + struct bpf_tramp_image im = {}; struct rv_jit_context ctx; int ret; diff --git a/arch/s390/crypto/hmac_s390.c b/arch/s390/crypto/hmac_s390.c index f8cd09f341d4..445fa7bbd958 100644 --- a/arch/s390/crypto/hmac_s390.c +++ b/arch/s390/crypto/hmac_s390.c @@ -150,7 +150,10 @@ static int hash_data(const u8 *in, unsigned int inlen, #undef PARAM_INIT - cpacf_klmd(func, ¶m, in, inlen); + if (final) + cpacf_klmd(func, ¶m, in, inlen); + else + cpacf_kimd(func, ¶m, in, inlen); memcpy(digest, ¶m, digestsize); diff --git a/arch/s390/include/asm/debug.h b/arch/s390/include/asm/debug.h index 39d484c59774..ad438d6352c8 100644 --- a/arch/s390/include/asm/debug.h +++ b/arch/s390/include/asm/debug.h @@ -460,7 +460,11 @@ static int VNAME(var, active_entries)[EARLY_AREAS] __initdata #define __REGISTER_STATIC_DEBUG_INFO(var, name, pages, areas, view) \ static int __init VNAME(var, reg)(void) \ { \ - debug_register_static(&var, (pages), (areas)); \ + int rc; \ + \ + rc = debug_register_static(&var, (pages), (areas)); \ + if (rc) \ + return rc; \ debug_register_view(&var, (view)); \ return 0; \ } \ @@ -493,7 +497,7 @@ static debug_info_t __refdata var = \ static debug_info_t __used __section(".s390dbf_info") *VNAME(var, info) = &var; \ __REGISTER_STATIC_DEBUG_INFO(var, name, pages, nr_areas, view) -void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas); +int debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas); #endif /* MODULE */ diff --git a/arch/s390/kernel/debug.c b/arch/s390/kernel/debug.c index b5bf8284dbfc..3b68f5c58333 100644 --- a/arch/s390/kernel/debug.c +++ b/arch/s390/kernel/debug.c @@ -434,7 +434,8 @@ static debug_info_t *debug_info_copy(debug_info_t *in, int mode) debug_info_free(rc); } while (1); - if (mode == NO_AREAS) + /* debug_register_static() failure leaves areas NULL, bounds intact */ + if (mode == NO_AREAS || !in->areas) goto out; for (i = 0; i < in->nr_areas; i++) { @@ -823,6 +824,9 @@ ssize_t debug_dump(debug_info_t *id, struct debug_view *view, file_private_info_t *p_info; size_t size, offset = 0; + if (!id) + return -EINVAL; + /* Need space for '\0' byte */ if (buf_size < 1) return 0; @@ -950,8 +954,12 @@ EXPORT_SYMBOL(debug_register); * * Note: This function is called automatically via an initcall generated by * DEFINE_STATIC_DEBUG_INFO. + * + * Return: + * - 0 on success + * - negative error code on failure */ -void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas) +int debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas) { unsigned long flags; debug_info_t *copy; @@ -959,7 +967,7 @@ void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas) if (!initialized) { pr_err("Tried to register debug feature %s too early\n", id->name); - return; + return -EINVAL; } debug_get_param(id->name, &id->level, &pages_per_area, false); @@ -975,7 +983,7 @@ void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas) id->active_entries = NULL; raw_spin_unlock_irqrestore(&id->lock, flags); - return; + return -ENOMEM; } /* Replace static trace area with dynamic copy. */ @@ -993,6 +1001,8 @@ void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas) mutex_lock(&debug_mutex); _debug_register(id); mutex_unlock(&debug_mutex); + + return 0; } /* Remove debugfs entries. */ diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c index dc14ebc0105b..8ea9dd7704ff 100644 --- a/arch/s390/kernel/uv.c +++ b/arch/s390/kernel/uv.c @@ -825,7 +825,7 @@ static int find_secret_in_page(const u8 secret_id[UV_SECRET_ID_LEN], { u16 i; - for (i = 0; i < list->total_num_secrets; i++) { + for (i = 0; i < list->num_secr_stored; i++) { if (memcmp(secret_id, list->secrets[i].id, UV_SECRET_ID_LEN) == 0) { *secret = list->secrets[i].hdr; return 0; @@ -846,11 +846,14 @@ int uv_find_secret(const u8 secret_id[UV_SECRET_ID_LEN], struct uv_secret_list *list, struct uv_secret_list_item_hdr *secret) { - u16 start_idx = 0; + u16 start_idx; u16 list_rc; int ret; + list->next_secret_idx = 0; + do { + start_idx = list->next_secret_idx; uv_list_secrets(list, start_idx, &list_rc, NULL); if (list_rc != UVC_RC_EXECUTED && list_rc != UVC_RC_MORE_DATA) { if (list_rc == UVC_RC_INV_CMD) @@ -861,7 +864,6 @@ int uv_find_secret(const u8 secret_id[UV_SECRET_ID_LEN], ret = find_secret_in_page(secret_id, list, secret); if (ret == 0) return ret; - start_idx = list->next_secret_idx; } while (list_rc == UVC_RC_MORE_DATA && start_idx < list->next_secret_idx); return -ENOENT; diff --git a/arch/s390/kernel/vdso/Makefile b/arch/s390/kernel/vdso/Makefile index 35c834b895ec..54bcf2984ca1 100644 --- a/arch/s390/kernel/vdso/Makefile +++ b/arch/s390/kernel/vdso/Makefile @@ -29,6 +29,7 @@ KBUILD_CFLAGS_VDSO := $(filter-out -munaligned-symbols,$(KBUILD_CFLAGS_VDSO)) KBUILD_CFLAGS_VDSO := $(filter-out -fno-asynchronous-unwind-tables,$(KBUILD_CFLAGS_VDSO)) KBUILD_CFLAGS_VDSO += -fPIC -fno-common -fno-builtin -fasynchronous-unwind-tables KBUILD_CFLAGS_VDSO += -fno-stack-protector $(DISABLE_KSTACK_ERASE) +KBUILD_CFLAGS_VDSO += $(CONFIG_CC_OPT_INLINE_MEMSET) ldflags-y := -shared -soname=linux-vdso.so.1 \ --hash-style=both --build-id=sha1 \ $(call ld-option, --eh-frame-hdr) -T diff --git a/arch/s390/kvm/gmap/dat.c b/arch/s390/kvm/gmap/dat.c index 24547e39fab2..ff80d02c9f56 100644 --- a/arch/s390/kvm/gmap/dat.c +++ b/arch/s390/kvm/gmap/dat.c @@ -621,17 +621,20 @@ int dat_get_storage_key(union asce asce, gfn_t gfn, union skey *skey) union pte *ptep; int rc; +again: skey->skey = 0; rc = dat_entry_walk(NULL, gfn, asce, DAT_WALK_ANY, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep); if (rc) return rc; if (!ptep) { - union crste crste; + union crste crste = READ_ONCE(*crstep); - crste = READ_ONCE(*crstep); - if (!crste.h.fc || !crste.s.fc1.pr) + if (!crste_leaf(crste) && !crste.h.i) + goto again; + if (!crste.s.fc1.pr) return 0; + skey->skey = page_get_storage_key(large_crste_to_phys(crste, gfn)); return 0; } @@ -662,13 +665,20 @@ int dat_set_storage_key(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gf union pte *ptep; int rc; +again: rc = dat_entry_walk(mc, gfn, asce, DAT_WALK_LEAF_ALLOC, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep); if (rc) return rc; if (!ptep) { - page_set_storage_key(large_crste_to_phys(*crstep, gfn), skey.skey, !nq); + union crste crste = READ_ONCE(*crstep); + + /* A large page has been split concurrently, try again */ + if (!crste_leaf(crste)) + goto again; + + page_set_storage_key(large_crste_to_phys(crste, gfn), skey.skey, !nq); return 0; } @@ -718,15 +728,22 @@ int dat_cond_set_storage_key(struct kvm_s390_mmu_cache *mmc, union asce asce, gf union pte *ptep; int rc; +again: rc = dat_entry_walk(mmc, gfn, asce, DAT_WALK_LEAF_ALLOC, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep); if (rc) return rc; if (!ptep) { + union crste crste = READ_ONCE(*crstep); + + /* A large page has been split concurrently, try again */ + if (!crste_leaf(crste)) + goto again; if (!oldkey) oldkey = &prev; - return page_cond_set_storage_key(large_crste_to_phys(*crstep, gfn), skey, oldkey, + + return page_cond_set_storage_key(large_crste_to_phys(crste, gfn), skey, oldkey, nq, mr, mc); } @@ -768,7 +785,7 @@ int dat_reset_reference_bit(union asce asce, gfn_t gfn, union skey *skey) int rc; skey->skey = 0; - +again: rc = dat_entry_walk(NULL, gfn, asce, DAT_WALK_ANY, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep); if (rc) return rc; @@ -776,9 +793,12 @@ int dat_reset_reference_bit(union asce asce, gfn_t gfn, union skey *skey) if (!ptep) { union crste crste = READ_ONCE(*crstep); - if (!crste.h.fc || !crste.s.fc1.pr) + /* A large page has been split concurrently, try again */ + if (!crste_leaf(crste) && !crste.h.i) + goto again; + if (!crste.s.fc1.pr) return 0; - skey->skey = page_reset_referenced(large_crste_to_phys(*crstep, gfn)) << 1; + skey->skey = page_reset_referenced(large_crste_to_phys(crste, gfn)) << 1; return 0; } old = pgste_get_lock(ptep); @@ -846,19 +866,12 @@ long dat_reset_skeys(union asce asce, gfn_t start) } #endif /* KVM_S390_MANAGES_S390_GUEST */ -struct slot_priv { - unsigned long token; - struct kvm_s390_mmu_cache *mc; -}; - static long _dat_slot_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) { - struct slot_priv *p = walk->priv; - union crste dummy = { .val = p->token }; union pte new_pte, pte = READ_ONCE(*ptep); union pgste pgste; - new_pte = _PTE_TOK(dummy.tok.type, dummy.tok.par); + new_pte = walk->priv ? _PTE_EMPTY : _PTE_TOK(_DAT_TOKEN_PIC, PGM_ADDRESSING); /* Table entry already in the desired state. */ if (pte.val == new_pte.val) @@ -875,10 +888,9 @@ static long _dat_slot_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_wal static long _dat_slot_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk) { union crste new_crste, crste = READ_ONCE(*crstep); - struct slot_priv *p = walk->priv; + struct kvm_s390_mmu_cache *mc = walk->priv; - new_crste.val = p->token; - new_crste.h.tt = crste.h.tt; + new_crste = mc ? _CRSTE_EMPTY(crste.h.tt) : _CRSTE_HOLE(crste.h.tt); /* Table entry already in the desired state. */ if (crste.val == new_crste.val) @@ -902,7 +914,10 @@ static long _dat_slot_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct d if (!crste.h.fc && !crste.h.i) return 0; /* Split (install a lower level table), and handle things there. */ - return dat_split_crste(p->mc, crstep, gfn, walk->asce, false); + if (mc) + return dat_split_crste(mc, crstep, gfn, walk->asce, false); + /* A large page should never cross memslots boundaries */ + return -EINVAL; } static const struct dat_walk_ops dat_slot_ops = { @@ -910,16 +925,10 @@ static const struct dat_walk_ops dat_slot_ops = { .crste_ops = { _dat_slot_crste, _dat_slot_crste, _dat_slot_crste, _dat_slot_crste, }, }; -int dat_set_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, gfn_t end, - u16 type, u16 param) +int dat_set_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, gfn_t end) { - struct slot_priv priv = { - .token = _CRSTE_TOK(0, type, param).val, - .mc = mc, - }; - return _dat_walk_gfn_range(start, end, asce, &dat_slot_ops, - DAT_WALK_IGN_HOLES | DAT_WALK_ANY, &priv); + DAT_WALK_IGN_HOLES | DAT_WALK_ANY, mc); } static void pgste_set_unlock_multiple(union pte *first, int n, union pgste *pgstes) diff --git a/arch/s390/kvm/gmap/dat.h b/arch/s390/kvm/gmap/dat.h index e452c141b841..90389d47ba4e 100644 --- a/arch/s390/kvm/gmap/dat.h +++ b/arch/s390/kvm/gmap/dat.h @@ -547,8 +547,7 @@ long dat_reset_skeys(union asce asce, gfn_t start); unsigned long dat_get_ptval(struct page_table *table, struct ptval_param param); void dat_set_ptval(struct page_table *table, struct ptval_param param, unsigned long val); -int dat_set_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, gfn_t end, - u16 type, u16 param); +int dat_set_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, gfn_t end); #if KVM_S390_MANAGES_S390_GUEST int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn); @@ -973,16 +972,15 @@ static inline int get_level(union crste *crstep, union pte *ptep) return ptep ? TABLE_TYPE_PAGE_TABLE : crstep->h.tt; } -static inline int dat_delete_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, - unsigned long npages) +static inline int dat_delete_slot(union asce asce, gfn_t start, unsigned long npages) { - return dat_set_slot(mc, asce, start, start + npages, _DAT_TOKEN_PIC, PGM_ADDRESSING); + return dat_set_slot(NULL, asce, start, start + npages); } static inline int dat_create_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, unsigned long npages) { - return dat_set_slot(mc, asce, start, start + npages, _DAT_TOKEN_NONE, 0); + return dat_set_slot(mc, asce, start, start + npages); } static inline bool crste_is_ucas(union crste crste) diff --git a/arch/s390/kvm/gmap/gmap.c b/arch/s390/kvm/gmap/gmap.c index 4968330e9553..3f3fa864cc36 100644 --- a/arch/s390/kvm/gmap/gmap.c +++ b/arch/s390/kvm/gmap/gmap.c @@ -994,11 +994,13 @@ static long _destroy_pages_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct da static long _destroy_pages_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk) { phys_addr_t origin, cur, end; + union crste crste; - if (!crstep->h.fc || !crstep->s.fc1.pr) + crste = READ_ONCE(*crstep); + if (!crste.h.fc || !crste.s.fc1.pr) return 0; - origin = crste_origin_large(*crstep); + origin = crste_origin_large(crste); cur = ((max(gfn, walk->start) - gfn) << PAGE_SHIFT) + origin; end = ((min(next, walk->end) - gfn) << PAGE_SHIFT) + origin; for ( ; cur < end; cur += PAGE_SIZE) diff --git a/arch/s390/kvm/gmap/kvm_mmu.c b/arch/s390/kvm/gmap/kvm_mmu.c index b08b8229bb6f..c2ffb5e59ec6 100644 --- a/arch/s390/kvm/gmap/kvm_mmu.c +++ b/arch/s390/kvm/gmap/kvm_mmu.c @@ -47,6 +47,9 @@ int s390_kvm_mmu_prepare_memory_region(struct kvm *kvm, struct kvm_memory_slot *new, enum kvm_mr_change change) { + struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL; + int rc = 0; + if (kvm_is_ucontrol(kvm) && new && new->id < KVM_USER_MEM_SLOTS) return -EINVAL; @@ -61,6 +64,10 @@ int s390_kvm_mmu_prepare_memory_region(struct kvm *kvm, * and munmap() stuff in this slot after doing this call at any * time. */ + if (change != KVM_MR_MOVE && change != KVM_MR_CREATE) { + WARN(1, "Unknown KVM MR CHANGE: %d\n", change); + return -EINVAL; + } if (new->userspace_addr & ~PAGE_MASK) return -EINVAL; if ((new->base_gfn + new->npages) * PAGE_SIZE > kvm->arch.mem_limit) @@ -69,65 +76,41 @@ int s390_kvm_mmu_prepare_memory_region(struct kvm *kvm, return -EINVAL; } - if (!kvm_s390_is_migration_mode(kvm)) - return 0; - - /* - * Turn off migration mode when: - * - userspace creates a new memslot with dirty logging off, - * - userspace modifies an existing memslot (MOVE or FLAGS_ONLY) and - * dirty logging is turned off. - * Migration mode expects dirty page logging being enabled to store - * its dirty bitmap. - */ - if (change != KVM_MR_DELETE && - !(new->flags & KVM_MEM_LOG_DIRTY_PAGES)) - WARN(kvm_s390_vm_stop_migration(kvm), - "Failed to stop migration mode"); - - return 0; -} - -void s390_kvm_mmu_commit_memory_region(struct kvm *kvm, - struct kvm_memory_slot *old, - const struct kvm_memory_slot *new, - enum kvm_mr_change change) -{ - struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL; - int rc = 0; - - guard(mutex)(&kvm->slots_arch_lock); + if (kvm->arch.migration_mode) { + /* + * Turn off migration mode when: + * - userspace creates a new memslot with dirty logging off, + * - userspace modifies an existing memslot (MOVE or FLAGS_ONLY) + * and dirty logging is turned off. + * Migration mode expects dirty page logging being enabled to + * store its dirty bitmap. + */ + if (change != KVM_MR_DELETE && + !(new->flags & KVM_MEM_LOG_DIRTY_PAGES)) + WARN(kvm_s390_vm_stop_migration(kvm), + "Failed to stop migration mode"); + } if (change == KVM_MR_FLAGS_ONLY) - return; - - mc = kvm_s390_new_mmu_cache(); - if (!mc) { - rc = -ENOMEM; - goto out; + return 0; + if (change != KVM_MR_DELETE) { + /* Enough capacity to add a new memslot */ + mc = kvm_s390_new_mmu_cache(); + if (!mc) + return -ENOMEM; } - scoped_guard(write_lock, &kvm->mmu_lock) { kvm_s390_update_cmma_dirty(kvm, old); - switch (change) { - case KVM_MR_DELETE: - rc = dat_delete_slot(mc, kvm->arch.gmap->asce, old->base_gfn, old->npages); - break; - case KVM_MR_MOVE: - rc = dat_delete_slot(mc, kvm->arch.gmap->asce, old->base_gfn, old->npages); - if (rc) - break; - fallthrough; - case KVM_MR_CREATE: + if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) + rc = dat_delete_slot(kvm->arch.gmap->asce, old->base_gfn, old->npages); + if (!rc && (change == KVM_MR_MOVE || change == KVM_MR_CREATE)) rc = dat_create_slot(mc, kvm->arch.gmap->asce, new->base_gfn, new->npages); - break; - case KVM_MR_FLAGS_ONLY: - break; - default: - WARN(1, "Unknown KVM MR CHANGE: %d\n", change); - } } -out: - if (rc) - pr_warn("failed to commit memory region\n"); + /* + * Can only be triggered if dat_{create,delete}_slot() found an + * internal inconsistency or if the mmu cache ran out of memory; + * both should be impossible. + */ + KVM_BUG_ON(rc, kvm); + return rc; } diff --git a/arch/s390/kvm/gmap/kvm_mmu.h b/arch/s390/kvm/gmap/kvm_mmu.h index cdbd390bd33c..43cde61bae03 100644 --- a/arch/s390/kvm/gmap/kvm_mmu.h +++ b/arch/s390/kvm/gmap/kvm_mmu.h @@ -10,9 +10,5 @@ int s390_kvm_mmu_prepare_memory_region(struct kvm *kvm, const struct kvm_memory_slot *old, struct kvm_memory_slot *new, enum kvm_mr_change change); -void s390_kvm_mmu_commit_memory_region(struct kvm *kvm, - struct kvm_memory_slot *old, - const struct kvm_memory_slot *new, - enum kvm_mr_change change); #endif /* ARCH_KVM_GMAP_KVM_MMU_H */ diff --git a/arch/s390/kvm/s390/gaccess.c b/arch/s390/kvm/s390/gaccess.c index e5c064f263df..405345ccc4f1 100644 --- a/arch/s390/kvm/s390/gaccess.c +++ b/arch/s390/kvm/s390/gaccess.c @@ -1589,12 +1589,25 @@ static inline int ___gaccess_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg parent = READ_ONCE(sg->parent); if (!parent) return -EAGAIN; +retry: scoped_guard(spinlock, &parent->children_lock) { if (READ_ONCE(sg->parent) != parent) return -EAGAIN; sg->invalidated = false; rc = _gaccess_do_shadow(vcpu->arch.mc, sg, saddr, walk); } + if (rc == -ENOENT) { + struct kvm_memory_slot *slot; + struct guest_fault *entries; + + entries = get_entries(walk); + slot = kvm_vcpu_gfn_to_memslot(vcpu, entries[LEVEL_MEM].gfn); + if (!slot) + return PGM_ADDRESSING; + rc = gmap_link(vcpu->arch.mc, parent, entries + LEVEL_MEM, slot); + if (!rc) + goto retry; + } if (!rc) kvm_s390_release_faultin_array(vcpu->kvm, walk->raw_entries, false); return rc; diff --git a/arch/s390/kvm/s390/interrupt.c b/arch/s390/kvm/s390/interrupt.c index 0381ae981703..f892f4307883 100644 --- a/arch/s390/kvm/s390/interrupt.c +++ b/arch/s390/kvm/s390/interrupt.c @@ -2984,61 +2984,58 @@ static int adapter_indicators_set(struct kvm *kvm, struct s390_io_adapter *adapter, struct kvm_s390_adapter_int *adapter_int) { - unsigned long bit; - int summary_set, idx; struct s390_map_info *ind_info, *summary_info; - void *map; struct page *ind_page, *summary_page; - unsigned long flags; + unsigned long bit; + int summary_set; + void *map; ind_page = NULL; - spin_lock_irqsave(&adapter->maps_lock, flags); - ind_info = get_map_info(adapter, adapter_int->ind_addr); + scoped_guard(spinlock_irqsave, &adapter->maps_lock) { + ind_info = get_map_info(adapter, adapter_int->ind_addr); + if (ind_info) { + map = page_address(ind_info->page); + bit = get_ind_bit(ind_info->addr, adapter_int->ind_offset, adapter->swap); + set_bit(bit, map); + } + } if (!ind_info) { - spin_unlock_irqrestore(&adapter->maps_lock, flags); ind_page = pin_map_page(kvm, adapter_int->ind_addr, 0); if (!ind_page) return -1; - idx = srcu_read_lock(&kvm->srcu); map = page_address(ind_page); bit = get_ind_bit(adapter_int->ind_addr, adapter_int->ind_offset, adapter->swap); set_bit(bit, map); - mark_page_dirty(kvm, adapter_int->ind_gaddr >> PAGE_SHIFT); set_page_dirty_lock(ind_page); - srcu_read_unlock(&kvm->srcu, idx); unpin_user_page(ind_page); - } else { - map = page_address(ind_info->page); - bit = get_ind_bit(ind_info->addr, adapter_int->ind_offset, adapter->swap); - set_bit(bit, map); - spin_unlock_irqrestore(&adapter->maps_lock, flags); } + scoped_guard(srcu, &kvm->srcu) + mark_page_dirty(kvm, gpa_to_gfn(adapter_int->ind_gaddr)); - spin_lock_irqsave(&adapter->maps_lock, flags); - summary_info = get_map_info(adapter, adapter_int->summary_addr); + scoped_guard(spinlock_irqsave, &adapter->maps_lock) { + summary_info = get_map_info(adapter, adapter_int->summary_addr); + if (summary_info) { + map = page_address(summary_info->page); + bit = get_ind_bit(summary_info->addr, adapter_int->summary_offset, + adapter->swap); + summary_set = test_and_set_bit(bit, map); + } + } if (!summary_info) { - spin_unlock_irqrestore(&adapter->maps_lock, flags); summary_page = pin_map_page(kvm, adapter_int->summary_addr, 0); if (!summary_page) return -1; - idx = srcu_read_lock(&kvm->srcu); map = page_address(summary_page); bit = get_ind_bit(adapter_int->summary_addr, adapter_int->summary_offset, adapter->swap); summary_set = test_and_set_bit(bit, map); - mark_page_dirty(kvm, adapter_int->summary_gaddr >> PAGE_SHIFT); set_page_dirty_lock(summary_page); - srcu_read_unlock(&kvm->srcu, idx); unpin_user_page(summary_page); - } else { - map = page_address(summary_info->page); - bit = get_ind_bit(summary_info->addr, adapter_int->summary_offset, - adapter->swap); - summary_set = test_and_set_bit(bit, map); - spin_unlock_irqrestore(&adapter->maps_lock, flags); } + scoped_guard(srcu, &kvm->srcu) + mark_page_dirty(kvm, gpa_to_gfn(adapter_int->summary_gaddr)); return summary_set ? 0 : 1; } @@ -3048,26 +3045,29 @@ static int adapter_indicators_set_fast(struct kvm *kvm, struct kvm_s390_adapter_int *adapter_int, int setbit) { + struct s390_map_info *ind_info, *summary_info; unsigned long bit; int summary_set; - struct s390_map_info *ind_info, *summary_info; void *map; - spin_lock(&adapter->maps_lock); + guard(srcu)(&kvm->srcu); + guard(spinlock)(&adapter->maps_lock); + ind_info = get_map_info(adapter, adapter_int->ind_addr); - if (!ind_info) { - spin_unlock(&adapter->maps_lock); + if (!ind_info) return -EWOULDBLOCK; - } + map = page_address(ind_info->page); bit = get_ind_bit(ind_info->addr, adapter_int->ind_offset, adapter->swap); - if (setbit) + if (setbit) { set_bit(bit, map); - summary_info = get_map_info(adapter, adapter_int->summary_addr); - if (!summary_info) { - spin_unlock(&adapter->maps_lock); - return -EWOULDBLOCK; + mark_page_dirty(kvm, gpa_to_gfn(adapter_int->ind_gaddr)); } + + summary_info = get_map_info(adapter, adapter_int->summary_addr); + if (!summary_info) + return -EWOULDBLOCK; + map = page_address(summary_info->page); bit = get_ind_bit(summary_info->addr, adapter_int->summary_offset, adapter->swap); @@ -3077,7 +3077,8 @@ static int adapter_indicators_set_fast(struct kvm *kvm, summary_set = test_and_set_bit(bit, map); else summary_set = test_and_clear_bit(bit, map); - spin_unlock(&adapter->maps_lock); + mark_page_dirty(kvm, gpa_to_gfn(adapter_int->summary_gaddr)); + return summary_set ? 0 : 1; } @@ -3228,9 +3229,9 @@ int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len break; } } - if (storestatus) { - n = kvm_s390_store_status_unloaded(vcpu, KVM_S390_STORE_STATUS_NOADDR); + scoped_guard(srcu, &vcpu->kvm->srcu) + n = kvm_s390_store_status_unloaded(vcpu, KVM_S390_STORE_STATUS_NOADDR); return r ? r : n; } diff --git a/arch/s390/kvm/s390/s390.c b/arch/s390/kvm/s390/s390.c index 5c73f43782a7..e4912db67adc 100644 --- a/arch/s390/kvm/s390/s390.c +++ b/arch/s390/kvm/s390/s390.c @@ -3579,12 +3579,11 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) { - mutex_lock(&vcpu->kvm->lock); preempt_disable(); vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch; vcpu->arch.sie_block->epdx = vcpu->kvm->arch.epdx; preempt_enable(); - mutex_unlock(&vcpu->kvm->lock); + if (!kvm_is_ucontrol(vcpu->kvm)) { vcpu->arch.gmap = vcpu->kvm->arch.gmap; sca_add_vcpu(vcpu); @@ -3757,13 +3756,11 @@ static int kvm_s390_vcpu_setup(struct kvm_vcpu *vcpu) kvm_s390_vcpu_pci_setup(vcpu); - mutex_lock(&vcpu->kvm->lock); if (kvm_s390_pv_is_protected(vcpu->kvm)) { rc = kvm_s390_pv_create_cpu(vcpu, &uvrc, &uvrrc); if (rc) kvm_s390_vcpu_unsetup_cmma(vcpu); } - mutex_unlock(&vcpu->kvm->lock); return rc; } @@ -5766,7 +5763,7 @@ static long cmma_d_count_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_ return 0; } -void kvm_s390_update_cmma_dirty(struct kvm *kvm, struct kvm_memory_slot *old) +void kvm_s390_update_cmma_dirty(struct kvm *kvm, const struct kvm_memory_slot *old) { const struct dat_walk_ops ops = { .pte_entry = cmma_d_count_pte, }; @@ -5781,7 +5778,6 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *old, const struct kvm_memory_slot *new, enum kvm_mr_change change) { - s390_kvm_mmu_commit_memory_region(kvm, old, new, change); } /** diff --git a/arch/s390/kvm/s390/s390.h b/arch/s390/kvm/s390/s390.h index d284a263ba70..aa0d1d062f8d 100644 --- a/arch/s390/kvm/s390/s390.h +++ b/arch/s390/kvm/s390/s390.h @@ -472,7 +472,7 @@ int __kvm_s390_mprotect_many(struct gmap *gmap, gpa_t gpa, u8 npages, unsigned i unsigned long bits); bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu); -void kvm_s390_update_cmma_dirty(struct kvm *kvm, struct kvm_memory_slot *old); +void kvm_s390_update_cmma_dirty(struct kvm *kvm, const struct kvm_memory_slot *old); int kvm_s390_vm_stop_migration(struct kvm *kvm); diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index c4b47070bb59..1b2566012b63 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -2566,6 +2566,8 @@ struct bpf_tramp_jit { int r14_off; /* Offset of saved %r14, has to be at the * bottom */ int do_fexit; /* do_fexit: label */ + int skip[BPF_MAX_TRAMP_LINKS]; /* skip: labels after each prog */ + int nr_progs; }; static void load_imm64(struct bpf_jit *jit, int dst_reg, u64 val) @@ -2584,6 +2586,7 @@ static void emit_store_stack_imm64(struct bpf_jit *jit, int tmp_reg, int stack_o } static int invoke_bpf_prog(struct bpf_tramp_jit *tjit, + struct bpf_tramp_image *im, const struct btf_func_model *m, struct bpf_tramp_node *node, bool save_ret) { @@ -2591,8 +2594,20 @@ static int invoke_bpf_prog(struct bpf_tramp_jit *tjit, int cookie_off = tjit->run_ctx_off + offsetof(struct bpf_tramp_run_ctx, bpf_cookie); struct bpf_prog *p = node->link->prog; + void *skip = jit->prg_buf + jit->prg; + int idx = tjit->nr_progs++; int patch; + if (idx >= ARRAY_SIZE(tjit->skip)) + return -E2BIG; + + /* + * nop, patched to skip this prog when it is detached + */ + + /* brcl 0,skip */ + EMIT6_PCREL_RILC(0xc0040000, 0, tjit->skip[idx]); + /* * run_ctx.cookie = node->cookie; */ @@ -2652,10 +2667,15 @@ static int invoke_bpf_prog(struct bpf_tramp_jit *tjit, /* brasl %r14,__bpf_prog_exit */ EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, bpf_trampoline_exit(p)); + /* skip: */ + tjit->skip[idx] = jit->prg; + bpf_tramp_image_add_skip(im, p, skip, jit->prg_buf + jit->prg); + return 0; } static int invoke_bpf(struct bpf_tramp_jit *tjit, + struct bpf_tramp_image *im, const struct btf_func_model *m, struct bpf_tramp_nodes *tn, bool save_ret, u64 func_meta, int cookie_off) @@ -2670,7 +2690,7 @@ static int invoke_bpf(struct bpf_tramp_jit *tjit, emit_store_stack_imm64(jit, REG_0, tjit->func_meta_off, meta); cur_cookie--; } - if (invoke_bpf_prog(tjit, m, tn->nodes[i], save_ret)) + if (invoke_bpf_prog(tjit, im, m, tn->nodes[i], save_ret)) return -EINVAL; } @@ -2712,6 +2732,11 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, u64 func_meta; int i, j; + /* The skip labels are taken from the previous pass. */ + tjit->nr_progs = 0; + if (im) + im->nr_skips = 0; + /* Support as many stack arguments as "mvc" instruction can handle. */ nr_reg_args = min_t(int, m->nr_args, MAX_NR_REG_ARGS); nr_stack_args = m->nr_args - nr_reg_args; @@ -2875,7 +2900,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, emit_store_stack_imm64(jit, REG_0, tjit->retval_off, 0); } - if (invoke_bpf(tjit, m, fentry, flags & BPF_TRAMP_F_RET_FENTRY_RET, + if (invoke_bpf(tjit, im, m, fentry, flags & BPF_TRAMP_F_RET_FENTRY_RET, func_meta, cookie_off)) return -EINVAL; @@ -2889,7 +2914,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, 0xf000 | tjit->retval_off); for (i = 0; i < fmod_ret->nr_nodes; i++) { - if (invoke_bpf_prog(tjit, m, fmod_ret->nodes[i], true)) + if (invoke_bpf_prog(tjit, im, m, fmod_ret->nodes[i], true)) return -EINVAL; /* @@ -2943,15 +2968,6 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, /* mvc tccnt_off(%r15),tail_call_cnt(4,%r15) */ _EMIT6(0xd203f000 | tjit->tccnt_off, 0xf000 | offsetof(struct prog_frame, tail_call_cnt)); - - im->ip_after_call = jit->prg_buf + jit->prg; - - /* - * The following nop will be patched by bpf_tramp_image_put(). - */ - - /* brcl 0,im->ip_epilogue */ - EMIT6_PCREL_RILC(0xc0040000, 0, (u64)im->ip_epilogue); } /* Set the "is_return" flag for fsession. */ @@ -2962,12 +2978,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, /* do_fexit: */ tjit->do_fexit = jit->prg; - if (invoke_bpf(tjit, m, fexit, false, func_meta, cookie_off)) + if (invoke_bpf(tjit, im, m, fexit, false, func_meta, cookie_off)) return -EINVAL; if (flags & BPF_TRAMP_F_CALL_ORIG) { - im->ip_epilogue = jit->prg_buf + jit->prg; - /* * __bpf_tramp_exit(im); */ @@ -3016,7 +3030,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, struct bpf_tramp_nodes *tnodes, void *orig_call) { - struct bpf_tramp_image im; + struct bpf_tramp_image im = {}; struct bpf_tramp_jit tjit; int ret; diff --git a/arch/s390/pci/pci_event.c b/arch/s390/pci/pci_event.c index f317a1465dad..d6af4015223e 100644 --- a/arch/s390/pci/pci_event.c +++ b/arch/s390/pci/pci_event.c @@ -226,6 +226,7 @@ static pci_ers_result_t zpci_event_attempt_error_recovery(struct pci_dev *pdev, device_lock(&pdev->dev); if (pdev->error_state == pci_channel_io_perm_failure) { ers_res = PCI_ERS_RESULT_DISCONNECT; + status_str = "skipped (permanent failure)"; goto out_unlock; } pdev->error_state = pci_channel_io_frozen; @@ -297,8 +298,8 @@ static pci_ers_result_t zpci_event_attempt_error_recovery(struct pci_dev *pdev, driver->err_handler->resume(pdev); pci_uevent_ers(pdev, PCI_ERS_RESULT_RECOVERED); out_unlock: + zpci_report_status(zdev, pdev, "recovery", status_str); device_unlock(&pdev->dev); - zpci_report_status(zdev, "recovery", status_str); return ers_res; } @@ -361,8 +362,10 @@ static void __zpci_event_error(struct zpci_ccdf_err *ccdf) __zpci_event_print_error(pdev, ccdf); - if (!pdev) + if (!pdev) { + zpci_report_status(zdev, NULL, "error event", "no pdev bound"); goto no_pdev; + } switch (ccdf->pec) { case 0x002a: /* Error event concerns FMB */ diff --git a/arch/s390/pci/pci_report.c b/arch/s390/pci/pci_report.c index 7030f7052926..72ecabf7003c 100644 --- a/arch/s390/pci/pci_report.c +++ b/arch/s390/pci/pci_report.c @@ -87,9 +87,23 @@ static struct debug_view debug_log_view = { NULL }; +static ssize_t zpci_report_pdev(struct pci_dev *pdev, char *buf, size_t size) +{ + struct pci_driver *driver; + const char *start = buf; + char *end = buf + size; + + device_lock_assert(&pdev->dev); + buf += scnprintf(buf, end - buf, "state: %s\n", zpci_state_str(pdev->error_state)); + driver = to_pci_driver(pdev->dev.driver); + buf += scnprintf(buf, end - buf, "driver: %s\n", (driver) ? driver->name : "n/a"); + return buf - start; +} + /** * zpci_report_status - Report the status of operations on a PCI device - * @zdev: The PCI device for which to report status + * @zdev: The zPCI device for which to report status + * @pdev: The PCI device associated with the zdev if any, NULL otherwise * @operation: A string representing the operation reported * @status: A string representing the status of the operation * @@ -103,15 +117,14 @@ static struct debug_view debug_log_view = { * * Return: 0 on success an error code < 0 otherwise. */ -int zpci_report_status(struct zpci_dev *zdev, const char *operation, const char *status) +int zpci_report_status(struct zpci_dev *zdev, struct pci_dev *pdev, + const char *operation, const char *status) { struct zpci_report_error *report; - struct pci_driver *driver = NULL; - struct pci_dev *pdev = NULL; char *buf, *end; int ret; - if (!zdev || !zdev->zbus) + if (!zdev) return -ENODEV; /* Protected virtualization hosts get nothing from us */ @@ -121,18 +134,13 @@ int zpci_report_status(struct zpci_dev *zdev, const char *operation, const char report = (void *)get_zeroed_page(GFP_KERNEL); if (!report) return -ENOMEM; - if (zdev->zbus->bus) - pdev = pci_get_slot(zdev->zbus->bus, zdev->devfn); - if (pdev) - driver = to_pci_driver(pdev->dev.driver); buf = report->data.log_data; end = report->data.log_data + ZPCI_REPORT_DATA_SIZE; buf += scnprintf(buf, end - buf, "report: %s\n", operation); buf += scnprintf(buf, end - buf, "status: %s\n", status); - buf += scnprintf(buf, end - buf, "state: %s\n", - (pdev) ? zpci_state_str(pdev->error_state) : "n/a"); - buf += scnprintf(buf, end - buf, "driver: %s\n", (driver) ? driver->name : "n/a"); + if (pdev) + buf += zpci_report_pdev(pdev, buf, end - buf); ret = debug_dump(pci_debug_msg_id, &debug_log_view, buf, end - buf, true); if (ret < 0) pr_err("Reading PCI debug messages failed with code %d\n", ret); diff --git a/arch/s390/pci/pci_report.h b/arch/s390/pci/pci_report.h index e08003d51a97..dd7b0b05001c 100644 --- a/arch/s390/pci/pci_report.h +++ b/arch/s390/pci/pci_report.h @@ -8,9 +8,11 @@ */ #ifndef __S390_PCI_REPORT_H #define __S390_PCI_REPORT_H +#include struct zpci_dev; -int zpci_report_status(struct zpci_dev *zdev, const char *operation, const char *status); +int zpci_report_status(struct zpci_dev *zdev, struct pci_dev *pdev, + const char *operation, const char *status); #endif /* __S390_PCI_REPORT_H */ diff --git a/arch/x86/coco/sev/svsm.c b/arch/x86/coco/sev/svsm.c index 916d62cd17dc..2d493d55ff2e 100644 --- a/arch/x86/coco/sev/svsm.c +++ b/arch/x86/coco/sev/svsm.c @@ -74,6 +74,14 @@ int svsm_perform_call_protocol(struct svsm_call *call) flags = native_local_irq_save(); + /* + * 'caa' is a per-CPU variable. To avoid using a stale or incorrect + * 'caa' if the task is preempted or migrated to another CPU after it + * is fetched, always fetch 'caa' and then issue the SVSM call with + * interrupts disabled. This ensures the correct 'caa' is used. + */ + call->caa = svsm_get_caa(); + ghcb = __sev_get_ghcb(&state); do { @@ -321,7 +329,6 @@ int snp_svsm_vtpm_send_command(u8 *buffer) { struct svsm_call call = {}; - call.caa = svsm_get_caa(); call.rax = SVSM_VTPM_CALL(SVSM_VTPM_CMD); call.rcx = __pa(buffer); @@ -345,7 +352,6 @@ bool snp_svsm_vtpm_probe(void) if (!snp_vmpl) return false; - call.caa = svsm_get_caa(); call.rax = SVSM_VTPM_CALL(SVSM_VTPM_QUERY); if (svsm_perform_call_protocol(&call)) diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index f522fff9231e..1903ed8bbe3b 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -1293,6 +1293,7 @@ static void gcm_process_assoc(const struct aes_gcm_key *key, u8 ghash_acc[16], unsigned int len; const u8 *src = walk.addr; + assoclen -= orig_len_this_step; if (unlikely(pos)) { len = min(len_this_step, 16 - pos); memcpy(&buf[pos], src, len); @@ -1320,7 +1321,6 @@ static void gcm_process_assoc(const struct aes_gcm_key *key, u8 ghash_acc[16], kernel_fpu_end(); kernel_fpu_begin(); } - assoclen -= orig_len_this_step; } if (unlikely(pos)) aes_gcm_aad_update(key, ghash_acc, buf, pos, flags); diff --git a/arch/x86/entry/vdso/vdso64/Makefile b/arch/x86/entry/vdso/vdso64/Makefile index 7c0790065b5e..c2353a065279 100644 --- a/arch/x86/entry/vdso/vdso64/Makefile +++ b/arch/x86/entry/vdso/vdso64/Makefile @@ -14,7 +14,7 @@ vobjs-$(CONFIG_X86_SGX) += vsgx.o vobjs-$(CONFIG_FUTEX_ROBUST_UNLOCK) += vfutex.o # Compilation flags -flags-y := -DBUILD_VDSO64 -m64 -mcmodel=small +flags-y := -DBUILD_VDSO64 -m64 -mcmodel=small $(CONFIG_CC_OPT_INLINE_MEMSET) # The location of this include matters! include $(src)/../common/Makefile.include diff --git a/arch/x86/events/amd/brs.c b/arch/x86/events/amd/brs.c index dc564688f3d7..54b13faba116 100644 --- a/arch/x86/events/amd/brs.c +++ b/arch/x86/events/amd/brs.c @@ -343,11 +343,10 @@ void amd_brs_drain(void) if (!amd_brs_match_plm(event, from, to)) continue; - perf_clear_branch_entry_bitfields(br+nr); - - br[nr].from = from; - br[nr].to = to; - + br[nr] = (struct perf_branch_entry){ + .from = from, + .to = to, + }; nr++; } empty: diff --git a/arch/x86/events/amd/lbr.c b/arch/x86/events/amd/lbr.c index 9d9c961989d5..a55646fcb846 100644 --- a/arch/x86/events/amd/lbr.c +++ b/arch/x86/events/amd/lbr.c @@ -184,13 +184,6 @@ void amd_pmu_lbr_read(void) entry.to.split.reserved) continue; - perf_clear_branch_entry_bitfields(br + out); - - br[out].from = sign_ext_branch_ip(entry.from.split.ip); - br[out].to = sign_ext_branch_ip(entry.to.split.ip); - br[out].mispred = entry.from.split.mispredict; - br[out].predicted = !br[out].mispred; - /* * Set branch speculation information using the status of * the valid and spec bits. @@ -208,7 +201,14 @@ void amd_pmu_lbr_read(void) * speculative and took the correct path */ idx = (entry.to.split.valid << 1) | entry.to.split.spec; - br[out].spec = lbr_spec_map[idx]; + + br[out] = (struct perf_branch_entry){ + .from = sign_ext_branch_ip(entry.from.split.ip), + .to = sign_ext_branch_ip(entry.to.split.ip), + .mispred = entry.from.split.mispredict, + .predicted = !entry.from.split.mispredict, + .spec = lbr_spec_map[idx], + }; out++; } diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 1ac2ca35db53..3ef80882843e 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -506,6 +506,8 @@ static struct event_constraint intel_pnc_event_constraints[] = { INTEL_EVENT_CONSTRAINT(0xce, 0x1), INTEL_UEVENT_CONSTRAINT(0x01b1, 0x8), + INTEL_UEVENT_CONSTRAINT(0x01b2, 0xf), + INTEL_UEVENT_CONSTRAINT(0x02b2, 0xf), INTEL_UEVENT_CONSTRAINT(0x0847, 0xf), INTEL_UEVENT_CONSTRAINT(0x0446, 0xf), INTEL_UEVENT_CONSTRAINT(0x0846, 0xf), @@ -520,6 +522,14 @@ static struct extra_reg intel_pnc_extra_regs[] __read_mostly = { INTEL_UEVENT_EXTRA_REG(0x022a, MSR_OMR_1, 0x40ffffff0000ffffull, OMR_1), INTEL_UEVENT_EXTRA_REG(0x042a, MSR_OMR_2, 0x40ffffff0000ffffull, OMR_2), INTEL_UEVENT_EXTRA_REG(0x082a, MSR_OMR_3, 0x40ffffff0000ffffull, OMR_3), + INTEL_UEVENT_EXTRA_REG(0x014f, MSR_OMR_0, 0x40ffffff0000ffffull, OMR_0), + INTEL_UEVENT_EXTRA_REG(0x024f, MSR_OMR_1, 0x40ffffff0000ffffull, OMR_1), + INTEL_UEVENT_EXTRA_REG(0x044f, MSR_OMR_2, 0x40ffffff0000ffffull, OMR_2), + INTEL_UEVENT_EXTRA_REG(0x084f, MSR_OMR_3, 0x40ffffff0000ffffull, OMR_3), + INTEL_UEVENT_EXTRA_REG(0x01d6, MSR_OMR_0, 0x40ffffff0000ffffull, OMR_0), + INTEL_UEVENT_EXTRA_REG(0x02d6, MSR_OMR_1, 0x40ffffff0000ffffull, OMR_1), + INTEL_UEVENT_EXTRA_REG(0x04d6, MSR_OMR_2, 0x40ffffff0000ffffull, OMR_2), + INTEL_UEVENT_EXTRA_REG(0x08d6, MSR_OMR_3, 0x40ffffff0000ffffull, OMR_3), INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd), INTEL_UEVENT_EXTRA_REG(0x02c6, MSR_PEBS_FRONTEND, 0x9, FE), INTEL_UEVENT_EXTRA_REG(0x03c6, MSR_PEBS_FRONTEND, 0x7fff1f, FE), @@ -5292,12 +5302,15 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data) struct kvm_pmu *kvm_pmu = (struct kvm_pmu *)data; u64 intel_ctrl = hybrid(cpuc->pmu, intel_ctrl); u64 pebs_mask = cpuc->pebs_enabled & x86_pmu.pebs_capable; - int global_ctrl, pebs_enable; + u64 guest_pebs_mask; + int global_ctrl; /* * In addition to obeying exclude_guest/exclude_host, remove bits being * used for PEBS when running a guest, because PEBS writes to virtual - * addresses (not physical addresses). + * addresses (not physical addresses). If the guest wants to utilize + * PEBS, and PEBS can be safely enabled in the guest, bits for the guest's + * PEBS-enabled counters will be OR'd back in as appropriate. */ *nr = 0; global_ctrl = (*nr)++; @@ -5327,41 +5340,68 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data) return arr; } - if (!kvm_pmu || !x86_pmu.pebs_ept) + /* + * If the CPU doesn't support PEBS in the guest, then there's nothing + * more to do as disabling PMCs via PERF_GLOBAL_CTRL is sufficient on + * CPUs with guest/host isolation. + */ + if (!x86_pmu.pebs_ept) return arr; + /* + * Restrict guest PEBS events to counters that (a) perf supports, (b) + * the guest wants to use for PEBS, (c) are not excluded from counting + * in the guest, and (d) _are_ excluded from counting in the host. + */ + guest_pebs_mask = pebs_mask & intel_ctrl & kvm_pmu->pebs_enable & + ~cpuc->intel_ctrl_host_mask & + cpuc->intel_ctrl_guest_mask; + + /* + * Disable counters where the guest PMC is different than the host PMC + * being used on behalf of the guest, as the PEBS record includes + * PERF_GLOBAL_STATUS, i.e. the guest will see overflow status for the + * wrong counter(s). + */ + guest_pebs_mask &= ~kvm_pmu->host_cross_mapped_mask; + + /* + * FIXME: Allow guest and host usage of PEBS events to co-exist instead + * of disabling guest PEBS entirely if the host is using PEBS. + * What exactly goes wrong if guest and host are using PEBS is + * unknown. + */ + if (pebs_mask & ~cpuc->intel_ctrl_guest_mask) + guest_pebs_mask = 0; + + /* + * Context switch DS_AREA and PEBS_DATA_CFG if and only if PEBS will be + * active in the guest; if no records will be generated while the guest + * is running, then simply keep the host values resident in hardware. + */ arr[(*nr)++] = (struct perf_guest_switch_msr){ .msr = MSR_IA32_DS_AREA, .host = (unsigned long)cpuc->ds, - .guest = kvm_pmu->ds_area, + .guest = guest_pebs_mask ? kvm_pmu->ds_area : (unsigned long)cpuc->ds, }; if (x86_pmu.intel_cap.pebs_baseline) { arr[(*nr)++] = (struct perf_guest_switch_msr){ .msr = MSR_PEBS_DATA_CFG, .host = cpuc->active_pebs_data_cfg, - .guest = kvm_pmu->pebs_data_cfg, + .guest = guest_pebs_mask ? kvm_pmu->pebs_data_cfg : + cpuc->active_pebs_data_cfg, }; } - pebs_enable = (*nr)++; - arr[pebs_enable] = (struct perf_guest_switch_msr){ - .msr = MSR_IA32_PEBS_ENABLE, - .host = cpuc->pebs_enabled & ~cpuc->intel_ctrl_guest_mask, - .guest = pebs_mask & ~cpuc->intel_ctrl_host_mask & kvm_pmu->pebs_enable, - }; - - if (arr[pebs_enable].host) { - /* Disable guest PEBS if host PEBS is enabled. */ - arr[pebs_enable].guest = 0; - } else { - /* Disable guest PEBS thoroughly for cross-mapped PEBS counters. */ - arr[pebs_enable].guest &= ~kvm_pmu->host_cross_mapped_mask; - arr[global_ctrl].guest &= ~kvm_pmu->host_cross_mapped_mask; - /* Set hw GLOBAL_CTRL bits for PEBS counter when it runs for guest */ - arr[global_ctrl].guest |= arr[pebs_enable].guest; - } - + /* + * Do NOT mess with PEBS_ENABLED. As above, disabling counters via + * PERF_GLOBAL_CTRL is sufficient, and loading a stale PEBS_ENABLED, + * e.g. on VM-Exit, can put the system in a bad state. Simply enable + * counters in PERF_GLOBAL_CTRL, as perf load PEBS_ENABLED with the + * full value, i.e. perf *also* relies on PERF_GLOBAL_CTRL. + */ + arr[global_ctrl].guest |= guest_pebs_mask; return arr; } @@ -6561,6 +6601,8 @@ static void intel_pmu_filter(struct pmu *pmu, int cpu, bool *ret) PMU_FORMAT_ATTR(offcore_rsp, "config1:0-63"); +PMU_FORMAT_ATTR(offmodule_rsp, "config1:0-63"); + PMU_FORMAT_ATTR(ldlat, "config1:0-15"); PMU_FORMAT_ATTR(frontend, "config1:0-23"); @@ -6609,6 +6651,20 @@ static struct attribute *skl_format_attr[] = { NULL, }; +static struct attribute *pnc_format_attr_rtm[] = { + &format_attr_in_tx.attr, + &format_attr_in_tx_cp.attr, + &format_attr_offmodule_rsp.attr, + &format_attr_ldlat.attr, + NULL +}; + +static struct attribute *pnc_format_attr[] = { + &format_attr_offmodule_rsp.attr, + &format_attr_ldlat.attr, + NULL +}; + static __initconst const struct x86_pmu core_pmu = { .name = "core", .handle_irq = x86_pmu_handle_irq, @@ -7474,6 +7530,7 @@ static struct attribute *adl_hybrid_tsx_attrs[] = { FORMAT_ATTR_HYBRID(in_tx, hybrid_big); FORMAT_ATTR_HYBRID(in_tx_cp, hybrid_big); FORMAT_ATTR_HYBRID(offcore_rsp, hybrid_big_small_tiny); +FORMAT_ATTR_HYBRID(offmodule_rsp, hybrid_big_small_tiny); FORMAT_ATTR_HYBRID(ldlat, hybrid_big_small_tiny); FORMAT_ATTR_HYBRID(frontend, hybrid_big); @@ -7512,6 +7569,23 @@ static struct attribute *mtl_hybrid_extra_attr[] = { NULL }; +static struct attribute *nvl_hybrid_extra_attr_rtm[] = { + ADL_HYBRID_RTM_FORMAT_ATTR, + FORMAT_HYBRID_PTR(offmodule_rsp), + FORMAT_HYBRID_PTR(ldlat), + FORMAT_HYBRID_PTR(frontend), + FORMAT_HYBRID_PTR(snoop_rsp), + NULL +}; + +static struct attribute *nvl_hybrid_extra_attr[] = { + FORMAT_HYBRID_PTR(offmodule_rsp), + FORMAT_HYBRID_PTR(ldlat), + FORMAT_HYBRID_PTR(frontend), + FORMAT_HYBRID_PTR(snoop_rsp), + NULL +}; + static bool is_attr_for_this_pmu(struct kobject *kobj, struct attribute *attr) { struct device *dev = kobj_to_dev(kobj); @@ -8549,6 +8623,8 @@ __init int intel_pmu_init(void) case INTEL_DIAMONDRAPIDS_X: intel_pmu_init_pnc(NULL); x86_pmu.pebs_latency_data = pnc_latency_data; + extra_attr = boot_cpu_has(X86_FEATURE_RTM) ? + pnc_format_attr_rtm : pnc_format_attr; pr_cont("Panthercove events, "); name = "panthercove"; @@ -8557,13 +8633,12 @@ __init int intel_pmu_init(void) glc_common: intel_pmu_init_glc(NULL); intel_pmu_pebs_data_source_skl(true); - + extra_attr = boot_cpu_has(X86_FEATURE_RTM) ? + hsw_format_attr : nhm_format_attr; glc_base: x86_pmu.pebs_ept = 1; x86_pmu.hw_config = hsw_hw_config; x86_pmu.get_event_constraints = glc_get_event_constraints; - extra_attr = boot_cpu_has(X86_FEATURE_RTM) ? - hsw_format_attr : nhm_format_attr; extra_skl_attr = skl_format_attr; mem_attr = glc_events_attrs; td_attr = glc_td_events_attrs; @@ -8785,7 +8860,7 @@ __init int intel_pmu_init(void) mem_attr = mtl_hybrid_mem_attrs; tsx_attr = adl_hybrid_tsx_attrs; extra_attr = boot_cpu_has(X86_FEATURE_RTM) ? - mtl_hybrid_extra_attr_rtm : mtl_hybrid_extra_attr; + nvl_hybrid_extra_attr_rtm : nvl_hybrid_extra_attr; /* Initialize big core specific PerfMon capabilities.*/ pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX]; @@ -8794,8 +8869,6 @@ __init int intel_pmu_init(void) /* Initialize Atom core specific PerfMon capabilities.*/ pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX]; intel_pmu_init_arw(&pmu->pmu); - - intel_pmu_pebs_data_source_lnl(); break; default: diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index b98029b44052..d0329a7eb8a5 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -277,8 +277,8 @@ static u64 pnc_pebs_l2_hit_data_source[PNC_PEBS_DATA_SOURCE_MAX] = { 0, /* 0x06: Reserved */ OP_LH | P(LVL, L2) | LEVEL(L2) | P(SNOOP, HIT), /* 0x07: L2 Hit Snoop HIT */ OP_LH | P(LVL, L2) | LEVEL(L2) | P(SNOOP, HITM), /* 0x08: L2 Hit Snoop Hit Modified */ - OP_LH | P(LVL, L2) | LEVEL(L2) | P(SNOOP, MISS), /* 0x09: Prefetch Promotion */ - OP_LH | P(LVL, L2) | LEVEL(L2) | P(SNOOP, MISS), /* 0x0a: Cross Core Prefetch Promotion */ + OP_LH | P(LVL, L2) | LEVEL(L2) | P(SNOOP, NONE), /* 0x09: Prefetch Promotion */ + OP_LH | P(LVL, L2) | LEVEL(L2) | P(SNOOP, NONE), /* 0x0a: Cross Core Prefetch Promotion */ 0, /* 0x0b: Reserved */ 0, /* 0x0c: Reserved */ 0, /* 0x0d: Reserved */ @@ -455,6 +455,7 @@ static inline void pebs_set_tlb_lock(u64 *val, bool tlb, bool lock) static u64 __grt_latency_data(struct perf_event *event, u64 status, u8 dse, bool tlb, bool lock, bool blk) { + union perf_mem_data_src src; u64 val; WARN_ON_ONCE(is_hybrid() && @@ -470,7 +471,16 @@ static u64 __grt_latency_data(struct perf_event *event, u64 status, else val |= P(BLK, NA); - return val; + src.val = val; + + if (event->hw.flags & + (PERF_X86_EVENT_PEBS_LDLAT | PERF_X86_EVENT_PEBS_LD_HSW)) + src.mem_op = P(OP, LOAD); + if (event->hw.flags & + (PERF_X86_EVENT_PEBS_STLAT | PERF_X86_EVENT_PEBS_ST_HSW)) + src.mem_op = P(OP, STORE); + + return src.val; } u64 grt_latency_data(struct perf_event *event, u64 status) @@ -528,7 +538,11 @@ static u64 arw_latency_data(struct perf_event *event, u64 status) val |= P(BLK, NA); src.val = val; - if (event->hw.flags & PERF_X86_EVENT_PEBS_ST_HSW) + if (event->hw.flags & + (PERF_X86_EVENT_PEBS_LDLAT | PERF_X86_EVENT_PEBS_LD_HSW)) + src.mem_op = P(OP, LOAD); + if (event->hw.flags & + (PERF_X86_EVENT_PEBS_STLAT | PERF_X86_EVENT_PEBS_ST_HSW)) src.mem_op = P(OP, STORE); return src.val; @@ -563,7 +577,11 @@ static u64 lnc_latency_data(struct perf_event *event, u64 status) val |= P(BLK, NA); src.val = val; - if (event->hw.flags & PERF_X86_EVENT_PEBS_ST_HSW) + if (event->hw.flags & + (PERF_X86_EVENT_PEBS_LDLAT | PERF_X86_EVENT_PEBS_LD_HSW)) + src.mem_op = P(OP, LOAD); + if (event->hw.flags & + (PERF_X86_EVENT_PEBS_STLAT | PERF_X86_EVENT_PEBS_ST_HSW)) src.mem_op = P(OP, STORE); return src.val; @@ -621,7 +639,11 @@ u64 pnc_latency_data(struct perf_event *event, u64 status) val |= P(BLK, NA); src.val = val; - if (event->hw.flags & PERF_X86_EVENT_PEBS_ST_HSW) + if (event->hw.flags & + (PERF_X86_EVENT_PEBS_LDLAT | PERF_X86_EVENT_PEBS_LD_HSW)) + src.mem_op = P(OP, LOAD); + if (event->hw.flags & + (PERF_X86_EVENT_PEBS_STLAT | PERF_X86_EVENT_PEBS_ST_HSW)) src.mem_op = P(OP, STORE); return src.val; @@ -1291,22 +1313,22 @@ struct event_constraint intel_glm_pebs_event_constraints[] = { struct event_constraint intel_grt_pebs_event_constraints[] = { /* Allow all events as PEBS with no flags */ - INTEL_HYBRID_LAT_CONSTRAINT(0x5d0, 0x3), - INTEL_HYBRID_LAT_CONSTRAINT(0x6d0, 0x3f), + INTEL_HYBRID_LDLAT_CONSTRAINT(0x5d0, 0x3), + INTEL_HYBRID_STLAT_CONSTRAINT(0x6d0, 0x3f), EVENT_CONSTRAINT_END }; struct event_constraint intel_cmt_pebs_event_constraints[] = { /* Allow all events as PEBS with no flags */ - INTEL_HYBRID_LAT_CONSTRAINT(0x5d0, 0x3), - INTEL_HYBRID_LAT_CONSTRAINT(0x6d0, 0xff), + INTEL_HYBRID_LDLAT_CONSTRAINT(0x5d0, 0x3), + INTEL_HYBRID_STLAT_CONSTRAINT(0x6d0, 0xff), EVENT_CONSTRAINT_END }; struct event_constraint intel_dkt_pebs_event_constraints[] = { /* Allow all events as PEBS with no flags */ - INTEL_HYBRID_LAT_CONSTRAINT(0x5d0, 0xff), - INTEL_HYBRID_LAT_CONSTRAINT(0x6d0, 0xff), + INTEL_HYBRID_LDLAT_CONSTRAINT(0x5d0, 0xff), + INTEL_HYBRID_STLAT_CONSTRAINT(0x6d0, 0xff), EVENT_CONSTRAINT_END }; @@ -1496,24 +1518,8 @@ struct event_constraint intel_lnc_pebs_event_constraints[] = { INTEL_FLAGS_UEVENT_CONSTRAINT(0x012a, 0x1), /* OCR.* events */ INTEL_FLAGS_UEVENT_CONSTRAINT(0x012b, 0x1), /* OCR.* events */ - INTEL_FLAGS_UEVENT_CONSTRAINT(0x04a4, 0x1), /* TOPDOWN.BAD_SPEC_SLOTS */ - INTEL_FLAGS_UEVENT_CONSTRAINT(0x08a4, 0x1), /* TOPDOWN.BR_MISPREDICT_SLOTS */ - INTEL_FLAGS_UEVENT_CONSTRAINT(0x10a4, 0x8), /* TOPDOWN.MEMORY_BOUND_SLOTS */ - INTEL_HYBRID_LDLAT_CONSTRAINT(0x1cd, 0x3fc), INTEL_HYBRID_STLAT_CONSTRAINT(0x2cd, 0x3), - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_LOADS */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x12d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_STORES */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x21d0, 0xf), /* MEM_INST_RETIRED.LOCK_LOADS */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x41d0, 0xf), /* MEM_INST_RETIRED.SPLIT_LOADS */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x42d0, 0xf), /* MEM_INST_RETIRED.SPLIT_STORES */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x81d0, 0xf), /* MEM_INST_RETIRED.ALL_LOADS */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x82d0, 0xf), /* MEM_INST_RETIRED.ALL_STORES */ - INTEL_FLAGS_UEVENT_CONSTRAINT(0x87d0, 0x3ff), /* MEM_INST_RETIRED.ANY */ - - INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD_RANGE(0xd1, 0xd4, 0xf), - - INTEL_FLAGS_EVENT_CONSTRAINT(0xd0, 0xf), /* * Everything else is handled by PMU_FL_PEBS_ALL, because we @@ -1526,18 +1532,6 @@ struct event_constraint intel_lnc_pebs_event_constraints[] = { struct event_constraint intel_pnc_pebs_event_constraints[] = { INTEL_HYBRID_LDLAT_CONSTRAINT(0x1cd, 0xfc), INTEL_HYBRID_STLAT_CONSTRAINT(0x2cd, 0x3), - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_LOADS */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x12d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_STORES */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x21d0, 0xf), /* MEM_INST_RETIRED.LOCK_LOADS */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x41d0, 0xf), /* MEM_INST_RETIRED.SPLIT_LOADS */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x42d0, 0xf), /* MEM_INST_RETIRED.SPLIT_STORES */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x81d0, 0xf), /* MEM_INST_RETIRED.ALL_LOADS */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x82d0, 0xf), /* MEM_INST_RETIRED.ALL_STORES */ - - INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD_RANGE(0xd1, 0xd4, 0xf), - - INTEL_FLAGS_EVENT_CONSTRAINT(0xd0, 0xf), - INTEL_FLAGS_EVENT_CONSTRAINT(0xd6, 0xf), /* * Everything else is handled by PMU_FL_PEBS_ALL, because we diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c index cbe5c762008d..22e2a06d5786 100644 --- a/arch/x86/events/intel/lbr.c +++ b/arch/x86/events/intel/lbr.c @@ -756,10 +756,10 @@ void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc) rdmsrq(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr); - perf_clear_branch_entry_bitfields(br); - - br->from = msr_lastbranch.from; - br->to = msr_lastbranch.to; + *br = (struct perf_branch_entry){ + .from = msr_lastbranch.from, + .to = msr_lastbranch.to, + }; br++; } cpuc->lbr_stack.nr = i; @@ -847,14 +847,15 @@ void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc) if (abort && x86_pmu.lbr_double_abort && out > 0) out--; - perf_clear_branch_entry_bitfields(br+out); - br[out].from = from; - br[out].to = to; - br[out].mispred = mis; - br[out].predicted = pred; - br[out].in_tx = in_tx; - br[out].abort = abort; - br[out].cycles = cycles; + br[out] = (struct perf_branch_entry){ + .from = from, + .to = to, + .mispred = mis, + .predicted = pred, + .in_tx = in_tx, + .abort = abort, + .cycles = cycles, + }; out++; } cpuc->lbr_stack.nr = out; @@ -905,6 +906,7 @@ static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc, struct perf_branch_entry *e; struct lbr_entry *lbr; u64 from, to, info; + bool mispred; int i; for (i = 0; i < x86_pmu.lbr_nr; i++) { @@ -921,24 +923,27 @@ static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc, to = rdlbr_to(i, lbr); info = rdlbr_info(i, lbr); - perf_clear_branch_entry_bitfields(e); + mispred = get_lbr_mispred(info); - e->from = from; - e->to = to; - e->mispred = get_lbr_mispred(info); - e->predicted = !e->mispred; - e->in_tx = !!(info & LBR_INFO_IN_TX); - e->abort = !!(info & LBR_INFO_ABORT); - e->cycles = get_lbr_cycles(info); - e->type = get_lbr_br_type(info); - - /* - * Leverage the reserved field of cpuc->lbr_entries[i] to - * temporarily store the branch counters information. - * The later code will decide what content can be disclosed - * to the perf tool. Pleae see intel_pmu_lbr_counters_reorder(). - */ - e->reserved = (info >> LBR_INFO_BR_CNTR_OFFSET) & LBR_INFO_BR_CNTR_FULL_MASK; + *e = (struct perf_branch_entry){ + .from = from, + .to = to, + .mispred = mispred, + .predicted = !mispred, + .in_tx = !!(info & LBR_INFO_IN_TX), + .abort = !!(info & LBR_INFO_ABORT), + .cycles = get_lbr_cycles(info), + .type = get_lbr_br_type(info), + /* + * Leverage the reserved field of + * cpuc->lbr_entries[i] to temporarily store the + * branch counters information. The later code will + * decide what content can be disclosed to the perf + * tool. Pleae see intel_pmu_lbr_counters_reorder(). + */ + .reserved = (info >> LBR_INFO_BR_CNTR_OFFSET) & + LBR_INFO_BR_CNTR_FULL_MASK, + }; } cpuc->lbr_stack.nr = i; diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index 4680cba91340..fab9da78a5c7 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -517,10 +517,6 @@ struct cpu_hw_events { __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \ HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST) -#define INTEL_HYBRID_LAT_CONSTRAINT(c, n) \ - __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \ - HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LAT_HYBRID) - #define INTEL_HYBRID_LDLAT_CONSTRAINT(c, n) \ __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \ HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LAT_HYBRID|PERF_X86_EVENT_PEBS_LD_HSW) diff --git a/arch/x86/include/uapi/asm/ptrace-abi.h b/arch/x86/include/uapi/asm/ptrace-abi.h index 5823584dea13..3656955c6faa 100644 --- a/arch/x86/include/uapi/asm/ptrace-abi.h +++ b/arch/x86/include/uapi/asm/ptrace-abi.h @@ -2,6 +2,7 @@ #ifndef _ASM_X86_PTRACE_ABI_H #define _ASM_X86_PTRACE_ABI_H +#if defined(__ASSEMBLER__) || defined(__FRAME_OFFSETS) #ifdef __i386__ #define EBX 0 @@ -25,7 +26,6 @@ #else /* __i386__ */ -#if defined(__ASSEMBLER__) || defined(__FRAME_OFFSETS) /* * C ABI says these regs are callee-preserved. They aren't saved on kernel entry * unless syscall needs a complete, fully filled "struct pt_regs". @@ -57,12 +57,12 @@ #define EFLAGS 144 #define RSP 152 #define SS 160 -#endif /* __ASSEMBLER__ */ /* top of stack page */ #define FRAME_SIZE 168 #endif /* !__i386__ */ +#endif /* defined(__ASSEMBLER__) || defined(__FRAME_OFFSETS) */ /* Arbitrarily choose the same ptrace numbers as used by the Sparc code. */ #define PTRACE_GETREGS 12 diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index ab469605fc89..39f238952e14 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -2108,6 +2108,9 @@ bool filter_mce(struct mce *m) static __always_inline void exc_machine_check_kernel(struct pt_regs *regs) { irqentry_state_t irq_state; + unsigned long dr7; + + dr7 = local_db_save(); WARN_ON_ONCE(user_mode(regs)); @@ -2116,20 +2119,26 @@ static __always_inline void exc_machine_check_kernel(struct pt_regs *regs) * mce_check_crashing_cpu() for details. */ if (mca_cfg.initialized && mce_check_crashing_cpu()) - return; + goto out; irq_state = irqentry_nmi_enter(regs); do_machine_check(regs); irqentry_nmi_exit(regs, irq_state); +out: + local_db_restore(dr7); } static __always_inline void exc_machine_check_user(struct pt_regs *regs) { + unsigned long dr7; + irqentry_enter_from_user_mode(regs); + dr7 = local_db_save(); do_machine_check(regs); + local_db_restore(dr7); irqentry_exit_to_user_mode(regs); } @@ -2138,21 +2147,13 @@ static __always_inline void exc_machine_check_user(struct pt_regs *regs) /* MCE hit kernel mode */ DEFINE_IDTENTRY_MCE(exc_machine_check) { - unsigned long dr7; - - dr7 = local_db_save(); exc_machine_check_kernel(regs); - local_db_restore(dr7); } /* The user mode variant. */ DEFINE_IDTENTRY_MCE_USER(exc_machine_check) { - unsigned long dr7; - - dr7 = local_db_save(); exc_machine_check_user(regs); - local_db_restore(dr7); } #ifdef CONFIG_X86_FRED @@ -2169,28 +2170,20 @@ DEFINE_IDTENTRY_MCE_USER(exc_machine_check) */ DEFINE_FREDENTRY_MCE(exc_machine_check) { - unsigned long dr7; - - dr7 = local_db_save(); if (user_mode(regs)) exc_machine_check_user(regs); else exc_machine_check_kernel(regs); - local_db_restore(dr7); } #endif #else /* 32bit unified entry point */ DEFINE_IDTENTRY_RAW(exc_machine_check) { - unsigned long dr7; - - dr7 = local_db_save(); if (user_mode(regs)) exc_machine_check_user(regs); else exc_machine_check_kernel(regs); - local_db_restore(dr7); } #endif diff --git a/arch/x86/kernel/sys_x86_64.c b/arch/x86/kernel/sys_x86_64.c index 776ae6fa7f2d..4c078827f26a 100644 --- a/arch/x86/kernel/sys_x86_64.c +++ b/arch/x86/kernel/sys_x86_64.c @@ -50,9 +50,16 @@ static unsigned long get_align_mask(struct file *filp) * value before calling vm_unmapped_area() or ORed directly to the * address. */ -static unsigned long get_align_bits(void) +static unsigned long get_align_bits(struct file *filp) { - return va_align.bits & get_align_mask(NULL); + /* + * va_align.bits is smaller than the huge page size and will + * lead to misaligned huge pages. Ignore it for huge mappings. + */ + if (is_file_hugepages(filp)) + return 0; + + return va_align.bits & get_align_mask(filp); } static int __init control_va_addr_alignment(char *str) @@ -157,7 +164,7 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, } if (filp) { info.align_mask = get_align_mask(filp); - info.align_offset += get_align_bits(); + info.align_offset += get_align_bits(filp); } return vm_unmapped_area(&info); @@ -222,7 +229,7 @@ arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr0, if (filp) { info.align_mask = get_align_mask(filp); - info.align_offset += get_align_bits(); + info.align_offset += get_align_bits(filp); } addr = vm_unmapped_area(&info); if (!(addr & ~PAGE_MASK)) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 064ecc33b926..a0d608e3fceb 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -2512,6 +2512,12 @@ static void shadow_walk_init_using_root(struct kvm_shadow_walk_iterator *iterato iterator->addr = addr; iterator->shadow_addr = root; + + if (WARN_ON_ONCE(!VALID_PAGE(root)) || kvm_mmu_is_dummy_root(root)) { + iterator->level = 0; + return; + } + iterator->level = vcpu->arch.mmu->root_role.level; if (iterator->level >= PT64_ROOT_4LEVEL && @@ -5058,7 +5064,7 @@ static int kvm_tdp_page_prefault(struct kvm_vcpu *vcpu, gpa_t gpa, if (signal_pending(current)) return -EINTR; - if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) + if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu)) return -EIO; cond_resched(); diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index a7d60c8785cd..d2fd47ee5ec8 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -823,14 +823,6 @@ void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu) bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) { - switch (msr) { - case MSR_CORE_PERF_GLOBAL_STATUS: - case MSR_CORE_PERF_GLOBAL_CTRL: - case MSR_CORE_PERF_GLOBAL_OVF_CTRL: - return kvm_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu)); - default: - break; - } return kvm_pmu_call(msr_idx_to_pmc)(vcpu, msr) || kvm_pmu_call(is_valid_msr)(vcpu, msr); } diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 73f37b050d0a..f9090b601efa 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -2125,13 +2125,8 @@ static bool svm_get_nested_state_pages(struct kvm_vcpu *vcpu) return false; } - if (!nested_svm_merge_msrpm(vcpu)) { - vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; - vcpu->run->internal.suberror = - KVM_INTERNAL_ERROR_EMULATION; - vcpu->run->internal.ndata = 0; + if (!nested_svm_merge_msrpm(vcpu)) return false; - } if (kvm_hv_verify_vp_assist(vcpu)) return false; diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 5705723f1f41..0c1ebb16cec6 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -488,6 +488,20 @@ static void snp_guest_req_cleanup(struct kvm *kvm) sev->guest_resp_buf = NULL; } +static int sev_alloc_have_run_cpus(struct kvm_sev_info *sev) +{ + if (!zalloc_cpumask_var(&sev->have_run_cpus, GFP_KERNEL_ACCOUNT)) + return -ENOMEM; + + return 0; +} + +static void sev_free_have_run_cpus(struct kvm_sev_info *sev) +{ + free_cpumask_var(sev->have_run_cpus); + memset(&sev->have_run_cpus, 0, sizeof(sev->have_run_cpus)); +} + static int __sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp, struct kvm_sev_init *data, unsigned long vm_type) @@ -545,10 +559,9 @@ static int __sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp, if (ret) goto e_free_asid; - if (!zalloc_cpumask_var(&sev->have_run_cpus, GFP_KERNEL_ACCOUNT)) { - ret = -ENOMEM; + ret = sev_alloc_have_run_cpus(sev); + if (ret) goto e_free_asid; - } /* This needs to happen after SEV/SNP firmware initialization. */ if (snp_active) { @@ -566,7 +579,7 @@ static int __sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp, return 0; e_free: - free_cpumask_var(sev->have_run_cpus); + sev_free_have_run_cpus(sev); e_free_asid: argp->error = init_args.error; sev_asid_free(sev); @@ -1125,9 +1138,6 @@ static int sev_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp) if (!sev_es_guest(kvm)) return -ENOTTY; - if (kvm_is_vcpu_creation_in_progress(kvm)) - return -EBUSY; - ret = kvm_lock_all_vcpus(kvm); if (ret) return ret; @@ -2035,6 +2045,13 @@ static void sev_migrate_from(struct kvm *dst_kvm, struct kvm *src_kvm) struct kvm_sev_info *mirror; unsigned long i; + /* + * Do cache maintenance on the source VM *before* clearing "SEV active", + * as memory reclaim flows won't trigger cache maintenance on the VM + * once it's no longer an SEV VM. + */ + sev_writeback_caches(src_kvm); + dst->active = true; dst->asid = src->asid; dst->handle = src->handle; @@ -2115,10 +2132,6 @@ static int sev_check_source_vcpus(struct kvm *dst, struct kvm *src) struct kvm_vcpu *src_vcpu; unsigned long i; - if (kvm_is_vcpu_creation_in_progress(src) || - kvm_is_vcpu_creation_in_progress(dst)) - return -EBUSY; - if (!sev_es_guest(src)) return 0; @@ -2187,11 +2200,14 @@ int sev_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd) * the set of CPUs from the source. If a CPU was used to run a vCPU in * the source VM but is never used for the destination VM, then the CPU * can only have cached memory that was accessible to the source VM. + * Furthermore, KVM *must* perform cache maintenance on the source VM, + * as the source VM may have access to memory that the destination VM + * does not, i.e. KVM could skip flushes if memory is reclaimed from + * the old VM but not the new VM. */ - if (!zalloc_cpumask_var(&dst_sev->have_run_cpus, GFP_KERNEL_ACCOUNT)) { - ret = -ENOMEM; + ret = sev_alloc_have_run_cpus(dst_sev); + if (ret) goto out_source_vcpu; - } sev_migrate_from(kvm, source_kvm); kvm_vm_dead(source_kvm); @@ -2510,9 +2526,6 @@ static int snp_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp) unsigned long i; int ret; - if (kvm_is_vcpu_creation_in_progress(kvm)) - return -EBUSY; - ret = kvm_lock_all_vcpus(kvm); if (ret) return ret; @@ -2888,10 +2901,9 @@ int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd) } mirror_sev = to_kvm_sev_info(kvm); - if (!zalloc_cpumask_var(&mirror_sev->have_run_cpus, GFP_KERNEL_ACCOUNT)) { - ret = -ENOMEM; + ret = sev_alloc_have_run_cpus(mirror_sev); + if (ret) goto e_unlock; - } /* * The mirror kvm holds an enc_context_owner ref so its asid can't @@ -2980,13 +2992,17 @@ void sev_vm_destroy(struct kvm *kvm) struct list_head *head = &sev->regions_list; struct list_head *pos, *q; + /* + * Free the mask even if the VM is not *currently* an SEV VM, as it may + * have been an SEV VM prior to intra-host migration. + */ + sev_free_have_run_cpus(sev); + if (!sev_guest(kvm)) return; WARN_ON(!list_empty(&sev->mirror_vms)); - free_cpumask_var(sev->have_run_cpus); - /* * If this is a mirror VM, remove it from the owner's list of a mirrors * and skip ASID cleanup (the ASID is tied to the lifetime of the owner). @@ -3616,7 +3632,6 @@ int pre_sev_run(struct vcpu_svm *svm, int cpu) sd->sev_vmcbs[asid] = svm->vmcb; svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID; - vmcb_mark_dirty(svm->vmcb, VMCB_ASID); return 0; } diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 7d59d301e1e5..fe0cba4731b2 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -673,16 +673,7 @@ static void clr_dr_intercepts(struct vcpu_svm *svm) static bool msr_write_intercepted(struct vcpu_svm *svm, u32 msr) { - /* - * For non-nested case: - * If the L01 MSR bitmap does not intercept the MSR, then we need to - * save it. - * - * For nested case: - * If the L02 MSR bitmap does not intercept the MSR, then we need to - * save it. - */ - void *msrpm = is_guest_mode(&svm->vcpu) ? svm->nested.msrpm : svm->msrpm; + void *msrpm = __va(__sme_clr(svm->vmcb->control.msrpm_base_pa)); return svm_test_msr_bitmap_write(msrpm, msr); } @@ -1902,8 +1893,7 @@ static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd) if (sd->next_asid > sd->max_asid) { ++sd->asid_generation; sd->next_asid = sd->min_asid; - svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID; - vmcb_mark_dirty(svm->vmcb, VMCB_ASID); + sd->flush_all_asids = true; } svm->current_vmcb->asid_generation = sd->asid_generation; @@ -4528,6 +4518,9 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) svm->vmcb->control.asid = svm->asid; vmcb_mark_dirty(svm->vmcb, VMCB_ASID); } + if (this_cpu_ptr(&svm_data)->flush_all_asids) + svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID; + svm->vmcb->save.cr2 = vcpu->arch.cr2; if (guest_cpu_cap_has(vcpu, X86_FEATURE_ERAPS) && @@ -4618,16 +4611,23 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) vcpu->arch.nested_run_pending = 0; } - svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; + if (!svm_is_vmrun_failure(svm->vmcb->control.exit_code)) { + this_cpu_ptr(&svm_data)->flush_all_asids = false; + svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; - /* - * Unconditionally mask off the CLEAR_RAP bit, the AND is just as cheap - * as the TEST+Jcc to avoid it. - */ - if (cpu_feature_enabled(X86_FEATURE_ERAPS)) - svm->vmcb->control.erap_ctl &= ~ERAP_CONTROL_CLEAR_RAP; + /* + * Unconditionally mask off the CLEAR_RAP bit, the AND is just + * as cheap as the TEST+Jcc to avoid it. + */ + if (cpu_feature_enabled(X86_FEATURE_ERAPS)) + svm->vmcb->control.erap_ctl &= ~ERAP_CONTROL_CLEAR_RAP; - vmcb_mark_all_clean(svm->vmcb); + vmcb_mark_all_clean(svm->vmcb); + + if (!msr_write_intercepted(svm, MSR_AMD64_PERF_CNTR_GLOBAL_CTL)) + rdmsrq(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, + vcpu_to_pmu(vcpu)->global_ctrl); + } /* if exit due to PF check for async PF */ if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) @@ -4636,9 +4636,6 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) kvm_clear_available_registers(vcpu, SVM_REGS_LAZY_LOAD_SET); - if (!msr_write_intercepted(svm, MSR_AMD64_PERF_CNTR_GLOBAL_CTL)) - rdmsrq(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, vcpu_to_pmu(vcpu)->global_ctrl); - trace_kvm_exit(vcpu, KVM_ISA_SVM); svm_complete_interrupts(vcpu); diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index e958943b8162..84f19026d3e8 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -376,6 +376,7 @@ struct svm_cpu_data { u32 next_asid; u32 min_asid; + bool flush_all_asids; bool bp_spec_reduce_set; struct vmcb *save_area; diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 151873407abd..b25216862740 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -1754,6 +1754,9 @@ static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx) static void copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx, u32 hv_clean_fields) { #ifdef CONFIG_KVM_HYPERV + const u64 runtime_controls = HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1 | + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2 | + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC; struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; struct hv_enlightened_vmcs *evmcs = nested_vmx_evmcs(vmx); struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(&vmx->vcpu); @@ -1762,6 +1765,9 @@ static void copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx, u32 hv_clean_fields vmcs12->tpr_threshold = evmcs->tpr_threshold; vmcs12->guest_rip = evmcs->guest_rip; + if ((hv_clean_fields & runtime_controls) != runtime_controls) + vmx->nested.force_msr_bitmap_recalc = true; + if (unlikely(!(hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_ENLIGHTENMENTSCONTROL))) { hv_vcpu->nested.pa_page_gpa = evmcs->partition_assist_page; @@ -3465,10 +3471,6 @@ static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu) } else { pr_debug_ratelimited("%s: no backing for APIC-access address in vmcs12\n", __func__); - vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; - vcpu->run->internal.suberror = - KVM_INTERNAL_ERROR_EMULATION; - vcpu->run->internal.ndata = 0; return false; } } @@ -3539,11 +3541,6 @@ static bool vmx_get_nested_state_pages(struct kvm_vcpu *vcpu) if (!nested_get_evmcs_page(vcpu)) { pr_debug_ratelimited("%s: enlightened vmptrld failed\n", __func__); - vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; - vcpu->run->internal.suberror = - KVM_INTERNAL_ERROR_EMULATION; - vcpu->run->internal.ndata = 0; - return false; } #endif @@ -3915,8 +3912,12 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) vmentry_failed: vcpu->arch.nested_run_pending = 0; - if (status == NVMX_VMENTRY_KVM_INTERNAL_ERROR) + if (status == NVMX_VMENTRY_KVM_INTERNAL_ERROR) { + vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; + vcpu->run->internal.ndata = 0; return 0; + } if (status == NVMX_VMENTRY_VMEXIT) return 1; WARN_ON_ONCE(status != NVMX_VMENTRY_VMFAIL); diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index bfa8612fb450..70a8c4816135 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -187,6 +187,9 @@ static bool intel_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) int ret; switch (msr) { + case MSR_CORE_PERF_GLOBAL_STATUS: + case MSR_CORE_PERF_GLOBAL_CTRL: + case MSR_CORE_PERF_GLOBAL_OVF_CTRL: case MSR_CORE_PERF_FIXED_CTR_CTRL: return kvm_pmu_has_perf_global_ctrl(pmu); case MSR_IA32_PEBS_ENABLE: diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index b272c20586a7..00850a3a77d6 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -1995,7 +1995,7 @@ static int tdx_handle_ept_violation(struct kvm_vcpu *vcpu) if (kvm_vcpu_has_events(vcpu) || signal_pending(current)) break; - if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) { + if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu)) { ret = -EIO; break; } @@ -2728,11 +2728,6 @@ static tdx_vm_state_guard_t tdx_acquire_vm_state_locks(struct kvm *kvm) mutex_lock(&kvm->lock); - if (kvm->created_vcpus != atomic_read(&kvm->online_vcpus)) { - r = -EBUSY; - goto out_err; - } - r = kvm_lock_all_vcpus(kvm); if (r) goto out_err; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 79468ddfe473..aad065d035fb 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -8060,7 +8060,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) bool req_immediate_exit = false; if (kvm_request_pending(vcpu)) { - if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) { + if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu)) { r = -EIO; goto out; } @@ -8072,6 +8072,10 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) { if (unlikely(!kvm_nested_call(get_nested_state_pages)(vcpu))) { + vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; + vcpu->run->internal.ndata = 0; + kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); r = 0; goto out; } diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c index cb03f5a2b243..4a105f283cfb 100644 --- a/arch/x86/mm/pgtable.c +++ b/arch/x86/mm/pgtable.c @@ -705,47 +705,37 @@ int pmd_clear_huge(pmd_t *pmd) } #ifdef CONFIG_X86_64 -/** - * pud_free_pmd_page - Clear PUD entry and free PMD page - * @pud: Pointer to a PUD - * @addr: Virtual address associated with PUD - * - * Context: The PUD range has been unmapped and TLB purged. - * Return: 1 if clearing the entry succeeded. 0 otherwise. - * - * NOTE: Callers must allow a single page allocation. +/* + * Given a PUD poitner, detach and free the pointed-to + * PMD page and any PTE page children. The entire range + * under the PUD must not have any valid translations + * and the TLB must have already been flushed. */ int pud_free_pmd_page(pud_t *pud, unsigned long addr) { - pmd_t *pmd, *pmd_sv; struct ptdesc *pt; + pmd_t *pmd; int i; pmd = pud_pgtable(*pud); - pmd_sv = (pmd_t *)__get_free_page(GFP_KERNEL); - if (!pmd_sv) - return 0; - - for (i = 0; i < PTRS_PER_PMD; i++) { - pmd_sv[i] = pmd[i]; - if (!pmd_none(pmd[i])) - pmd_clear(&pmd[i]); - } + /* Detach the PMD page: */ pud_clear(pud); - /* INVLPG to clear all paging-structure caches */ + /* + * PMD and all its descendents are unreachable + * via normal page walks. Make them unreachable + * in cached mid-level walks too: + */ flush_tlb_kernel_range(addr, addr + PAGE_SIZE-1); for (i = 0; i < PTRS_PER_PMD; i++) { - if (!pmd_none(pmd_sv[i])) { - pt = page_ptdesc(pmd_page(pmd_sv[i])); + if (!pmd_none(pmd[i])) { + pt = page_ptdesc(pmd_page(pmd[i])); pagetable_dtor_free(pt); } } - free_page((unsigned long)pmd_sv); - pmd_free(&init_mm, pmd); return 1; diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 2853e87797a7..6bca87457e87 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -3217,16 +3217,21 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog, } static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, + struct bpf_tramp_image *im, struct bpf_tramp_node *node, int stack_size, int run_ctx_off, bool save_ret, void *image, void *rw_image) { u8 *prog = *pprog; - u8 *jmp_insn; + u8 *jmp_insn, *skip; int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); struct bpf_prog *p = node->link->prog; u64 cookie = node->cookie; + /* nop, patched to skip this prog when it is detached */ + skip = image + (prog - (u8 *)rw_image); + emit_nops(&prog, X86_PATCH_SIZE); + /* mov rdi, cookie */ emit_mov_imm64(&prog, BPF_REG_1, (long) cookie >> 32, (u32) (long) cookie); @@ -3301,6 +3306,8 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, if (emit_rsb_call(&prog, bpf_trampoline_exit(p), image + (prog - (u8 *)rw_image))) return -EINVAL; + bpf_tramp_image_add_skip(im, p, skip, image + (prog - (u8 *)rw_image)); + *pprog = prog; return 0; } @@ -3332,6 +3339,7 @@ static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond) } static int invoke_bpf(const struct btf_func_model *m, u8 **pprog, + struct bpf_tramp_image *im, struct bpf_tramp_nodes *tl, int stack_size, int run_ctx_off, int func_meta_off, bool save_ret, void *image, void *rw_image, u64 func_meta, @@ -3346,7 +3354,7 @@ static int invoke_bpf(const struct btf_func_model *m, u8 **pprog, func_meta | (cur_cookie << BPF_TRAMP_COOKIE_INDEX_SHIFT)); cur_cookie--; } - if (invoke_bpf_prog(m, &prog, tl->nodes[i], stack_size, + if (invoke_bpf_prog(m, &prog, im, tl->nodes[i], stack_size, run_ctx_off, save_ret, image, rw_image)) return -EINVAL; } @@ -3355,6 +3363,7 @@ static int invoke_bpf(const struct btf_func_model *m, u8 **pprog, } static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, + struct bpf_tramp_image *im, struct bpf_tramp_nodes *tl, int stack_size, int run_ctx_off, u8 **branches, void *image, void *rw_image) @@ -3368,7 +3377,7 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, emit_mov_imm32(&prog, false, BPF_REG_0, 0); emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); for (i = 0; i < tl->nr_nodes; i++) { - if (invoke_bpf_prog(m, &prog, tl->nodes[i], stack_size, run_ctx_off, true, + if (invoke_bpf_prog(m, &prog, im, tl->nodes[i], stack_size, run_ctx_off, true, image, rw_image)) return -EINVAL; @@ -3640,7 +3649,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im } if (fentry->nr_nodes) { - if (invoke_bpf(m, &prog, fentry, regs_off, run_ctx_off, func_meta_off, + if (invoke_bpf(m, &prog, im, fentry, regs_off, run_ctx_off, func_meta_off, flags & BPF_TRAMP_F_RET_FENTRY_RET, image, rw_image, func_meta, cookie_off)) return -EINVAL; @@ -3652,7 +3661,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im if (!branches) return -ENOMEM; - if (invoke_bpf_mod_ret(m, &prog, fmod_ret, regs_off, + if (invoke_bpf_mod_ret(m, &prog, im, fmod_ret, regs_off, run_ctx_off, branches, image, rw_image)) { ret = -EINVAL; goto cleanup; @@ -3682,8 +3691,6 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im } /* remember return value in a stack for bpf prog to access */ emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); - im->ip_after_call = image + (prog - (u8 *)rw_image); - emit_nops(&prog, X86_PATCH_SIZE); } if (fmod_ret->nr_nodes) { @@ -3708,7 +3715,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im emit_store_stack_imm64(&prog, BPF_REG_0, -func_meta_off, func_meta); if (fexit->nr_nodes) { - if (invoke_bpf(m, &prog, fexit, regs_off, run_ctx_off, func_meta_off, + if (invoke_bpf(m, &prog, im, fexit, regs_off, run_ctx_off, func_meta_off, false, image, rw_image, func_meta, cookie_off)) { ret = -EINVAL; goto cleanup; @@ -3723,7 +3730,6 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im * restored to R0. */ if (flags & BPF_TRAMP_F_CALL_ORIG) { - im->ip_epilogue = image + (prog - (u8 *)rw_image); /* arg1: mov rdi, im */ emit_mov_imm64(&prog, BPF_REG_1, (long) im >> 32, (u32) (long) im); if (emit_rsb_call(&prog, __bpf_tramp_exit, image + (prog - (u8 *)rw_image))) { @@ -3811,7 +3817,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, struct bpf_tramp_nodes *tnodes, void *func_addr) { - struct bpf_tramp_image im; + struct bpf_tramp_image im = {}; void *image; int ret; diff --git a/arch/x86/pci/fixup.c b/arch/x86/pci/fixup.c index b301c6c8df75..795d81c7a4de 100644 --- a/arch/x86/pci/fixup.c +++ b/arch/x86/pci/fixup.c @@ -886,6 +886,105 @@ static void quirk_clear_strap_no_soft_reset_dev2_f0(struct pci_dev *dev) } } DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, 0x15b8, quirk_clear_strap_no_soft_reset_dev2_f0); + +/* + * Enhanced atomic operations can cause corruption with 64-bit DMA + * on these devices. + */ +#define RX_ENH_ATOMIC_EN BIT(8) + +static const u32 nbio_7_7_pcie_smn_addrs[] = { + 0x111401d0, + 0x111411d0, + 0x111421d0, + 0x111431d0, + 0x111441d0, + 0x112401d0, + 0x112411d0, + 0x112421d0, + 0x112431d0, + 0x112441d0, + 0x112451d0, + 0x113401d0, + 0x114401d0, +}; + +static const u32 nbio_7_11_pcie_smn_addrs[] = { + 0x112401d0, + 0x112411d0, + 0x112421d0, + 0x112431d0, + 0x112441d0, + 0x112451d0, + 0x113401d0, + 0x113411d0, + 0x113421d0, + 0x113431d0, + 0x113441d0, + 0x113451d0, +}; + +static void quirk_amd_nbio_enhanced_atomic(struct pci_dev *host_bridge, + const u32 *smn_addrs, + size_t nr_smn_addrs) +{ + bool changed = false; + size_t i; + u32 data; + int ret; + + for (i = 0; i < nr_smn_addrs; i++) { + ret = amd_smn_read(0, smn_addrs[i], &data); + if (ret) + continue; + if (!(data & RX_ENH_ATOMIC_EN)) + continue; + data = data & ~RX_ENH_ATOMIC_EN; + ret = amd_smn_write(0, smn_addrs[i], data); + if (ret) + continue; + if (changed) + continue; + ret = amd_smn_read(0, smn_addrs[i], &data); + if (ret) + continue; + if (data & RX_ENH_ATOMIC_EN) + continue; + changed = true; + } + + if (changed) + pci_info(host_bridge, "enhanced atomics disabled\n"); +} + +static void quirk_amd_nbio_7_7_disable_enhanced_atomic(struct pci_dev *dev) +{ + quirk_amd_nbio_enhanced_atomic(dev, nbio_7_7_pcie_smn_addrs, + ARRAY_SIZE(nbio_7_7_pcie_smn_addrs)); +} + +static void quirk_amd_nbio_7_11_disable_enhanced_atomic(struct pci_dev *dev) +{ + quirk_amd_nbio_enhanced_atomic(dev, nbio_7_11_pcie_smn_addrs, + ARRAY_SIZE(nbio_7_11_pcie_smn_addrs)); +} + +/* Phoenix, Hawk Point (NBIO 7.7) */ +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, 0x14E8, + quirk_amd_nbio_7_7_disable_enhanced_atomic); +DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_AMD, 0x14E8, + quirk_amd_nbio_7_7_disable_enhanced_atomic); + +/* Strix, Krackan, Strix Halo (NBIO 7.11) */ +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, 0x1507, + quirk_amd_nbio_7_11_disable_enhanced_atomic); +DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_AMD, 0x1507, + quirk_amd_nbio_7_11_disable_enhanced_atomic); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, 0x1122, + quirk_amd_nbio_7_11_disable_enhanced_atomic); +DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_AMD, 0x1122, + quirk_amd_nbio_7_11_disable_enhanced_atomic); + #endif /* diff --git a/arch/x86/um/asm/ptrace.h b/arch/x86/um/asm/ptrace.h index 2641d28d115c..439c4151f6b7 100644 --- a/arch/x86/um/asm/ptrace.h +++ b/arch/x86/um/asm/ptrace.h @@ -13,9 +13,7 @@ enum { }; #include -#ifndef CONFIG_X86_32 -#define __FRAME_OFFSETS /* Needed to get the R* macros */ -#endif +#define __FRAME_OFFSETS /* Needed to get the register macros */ #include #define user_mode(r) UPT_IS_USER(&(r)->regs) diff --git a/arch/x86/um/ptrace_32.c b/arch/x86/um/ptrace_32.c index 3af3cb821524..9e9155b0e918 100644 --- a/arch/x86/um/ptrace_32.c +++ b/arch/x86/um/ptrace_32.c @@ -7,6 +7,7 @@ #include #include #include +#define __FRAME_OFFSETS #include #include #include diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index 2b5c29434e42..b56db1cc6778 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -2091,6 +2091,7 @@ static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio, struct request_queue *q = bio->bi_bdev->bd_queue; struct blkcg *blkcg = css_to_blkcg(css); struct blkcg_gq *blkg; + unsigned long flags; rcu_read_lock(); blkg = blkg_lookup(blkcg, q); @@ -2105,11 +2106,11 @@ static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio, * Fast path failed, we're probably issuing IO in this cgroup the first * time, hold lock to create new blkg. */ - spin_lock_irq(&q->queue_lock); + spin_lock_irqsave(&q->queue_lock, flags); blkg = blkg_lookup_create(blkcg, bio->bi_bdev->bd_disk); if (blkg) blkg = blkg_lookup_tryget(blkg); - spin_unlock_irq(&q->queue_lock); + spin_unlock_irqrestore(&q->queue_lock, flags); return blkg; } diff --git a/block/blk-mq.c b/block/blk-mq.c index a26a11c73ee3..025a799f35b9 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -447,16 +447,6 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, WRITE_ONCE(rq->deadline, 0); req_ref_set(rq, 1); - if (rq->rq_flags & RQF_USE_SCHED) { - struct elevator_queue *e = data->q->elevator; - - INIT_HLIST_NODE(&rq->hash); - RB_CLEAR_NODE(&rq->rb_node); - - if (e->type->ops.prepare_request) - e->type->ops.prepare_request(rq); - } - return rq; } @@ -498,6 +488,12 @@ __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data) return rq_list_pop(data->cached_rqs); } +static bool blk_op_bypass_sched(blk_opf_t opf) +{ + return (opf & REQ_OP_MASK) == REQ_OP_FLUSH || + blk_op_is_passthrough(opf); +} + static void blk_mq_limit_depth(struct blk_mq_alloc_data *data) { struct elevator_mq_ops *ops; @@ -518,12 +514,10 @@ static void blk_mq_limit_depth(struct blk_mq_alloc_data *data) * Flush/passthrough requests are special and go directly to the * dispatch list, they are not subject to the async_depth limit. */ - if ((data->cmd_flags & REQ_OP_MASK) == REQ_OP_FLUSH || - blk_op_is_passthrough(data->cmd_flags)) + if (blk_op_bypass_sched(data->cmd_flags)) return; WARN_ON_ONCE(data->flags & BLK_MQ_REQ_RESERVED); - data->rq_flags |= RQF_USE_SCHED; /* * By default, sync requests have no limit, and async requests are @@ -534,6 +528,29 @@ static void blk_mq_limit_depth(struct blk_mq_alloc_data *data) ops->limit_depth(data->cmd_flags, data); } +/* + * Finish initializing a request once it has been claimed for an operation. + * Cached requests are allocated before that operation is known. + */ +static void blk_mq_rq_late_init(struct request *rq, u64 alloc_time_ns) +{ + struct elevator_queue *e; + + blk_mq_rq_time_init(rq, alloc_time_ns); + + if (!(rq->rq_flags & RQF_SCHED_TAGS) || (rq->rq_flags & RQF_RESV) || + blk_op_bypass_sched(rq->cmd_flags)) + return; + + rq->rq_flags |= RQF_USE_SCHED; + INIT_HLIST_NODE(&rq->hash); + RB_CLEAR_NODE(&rq->rb_node); + + e = rq->q->elevator; + if (e->type->ops.prepare_request) + e->type->ops.prepare_request(rq); +} + static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data) { struct request_queue *q = data->q; @@ -562,7 +579,7 @@ static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data) if (data->nr_tags > 1) { rq = __blk_mq_alloc_requests_batch(data); if (rq) { - blk_mq_rq_time_init(rq, alloc_time_ns); + blk_mq_rq_late_init(rq, alloc_time_ns); return rq; } data->nr_tags = 1; @@ -590,7 +607,7 @@ static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data) if (!(data->rq_flags & RQF_SCHED_TAGS)) blk_mq_inc_active_requests(data->hctx); rq = blk_mq_rq_ctx_init(data, blk_mq_tags_from_data(data), tag); - blk_mq_rq_time_init(rq, alloc_time_ns); + blk_mq_rq_late_init(rq, alloc_time_ns); return rq; } @@ -637,8 +654,6 @@ static struct request *blk_mq_alloc_cached_request(struct request_queue *q, if (plug->nr_ios == 1) return NULL; rq = blk_mq_rq_cache_fill(q, plug, opf, flags); - if (!rq) - return NULL; } else { rq = rq_list_peek(&plug->cached_rqs); if (!rq || rq->q != q) @@ -646,15 +661,13 @@ static struct request *blk_mq_alloc_cached_request(struct request_queue *q, if (blk_mq_get_hctx_type(opf) != rq->mq_hctx->type) return NULL; - if (op_is_flush(rq->cmd_flags) != op_is_flush(opf)) - return NULL; rq_list_pop(&plug->cached_rqs); - blk_mq_rq_time_init(rq, blk_time_get_ns()); + rq->cmd_flags = opf; + INIT_LIST_HEAD(&rq->queuelist); + blk_mq_rq_late_init(rq, blk_time_get_ns()); } - rq->cmd_flags = opf; - INIT_LIST_HEAD(&rq->queuelist); return rq; } @@ -766,7 +779,7 @@ struct request *blk_mq_alloc_request_hctx(struct request_queue *q, if (!(data.rq_flags & RQF_SCHED_TAGS)) blk_mq_inc_active_requests(data.hctx); rq = blk_mq_rq_ctx_init(&data, blk_mq_tags_from_data(&data), tag); - blk_mq_rq_time_init(rq, alloc_time_ns); + blk_mq_rq_late_init(rq, alloc_time_ns); rq->__data_len = 0; rq->phys_gap_bit = 0; rq->__sector = (sector_t) -1; @@ -3060,8 +3073,6 @@ static struct request *blk_mq_get_cached_request(struct blk_plug *plug, if (type != rq->mq_hctx->type && (type != HCTX_TYPE_READ || rq->mq_hctx->type != HCTX_TYPE_DEFAULT)) return NULL; - if (op_is_flush(rq->cmd_flags) != op_is_flush(opf)) - return NULL; rq_list_pop(&plug->cached_rqs); return rq; } @@ -3163,9 +3174,9 @@ void blk_mq_submit_bio(struct bio *bio) new_request: if (rq) { rq_qos_throttle(rq->q, bio); - blk_mq_rq_time_init(rq, blk_time_get_ns()); rq->cmd_flags = bio->bi_opf; INIT_LIST_HEAD(&rq->queuelist); + blk_mq_rq_late_init(rq, blk_time_get_ns()); } else { rq = blk_mq_get_new_requests(q, plug, bio); if (unlikely(!rq)) { diff --git a/block/blk-zoned.c b/block/blk-zoned.c index a5afb842bf35..475aa16bc41a 100644 --- a/block/blk-zoned.c +++ b/block/blk-zoned.c @@ -2018,12 +2018,17 @@ static int disk_revalidate_zone_resources(struct gendisk *disk, struct blk_revalidate_zone_args *args) { struct queue_limits *lim = &disk->queue->limits; + unsigned long long nr_zones; unsigned int pool_size; int ret = 0; args->disk = disk; - args->nr_zones = - DIV_ROUND_UP_ULL(get_capacity(disk), lim->chunk_sectors); + nr_zones = DIV_ROUND_UP_ULL(get_capacity(disk), lim->chunk_sectors); + if (nr_zones > UINT_MAX) { + pr_warn("%s: Too many zones (%llu)\n", disk->disk_name, nr_zones); + return -EINVAL; + } + args->nr_zones = nr_zones; /* Cached zone conditions: 1 byte per zone */ args->zones_cond = kzalloc(args->nr_zones, GFP_NOIO); @@ -2131,6 +2136,12 @@ static int blk_revalidate_zone_cond(struct blk_zone *zone, unsigned int idx, { enum blk_zone_cond cond = zone->cond; + if (idx >= args->nr_zones) { + pr_warn("%s: Zone report index %u exceeds zone count %u\n", + args->disk->disk_name, idx, args->nr_zones); + return -EINVAL; + } + /* Check that the zone condition is consistent with the zone type. */ switch (cond) { case BLK_ZONE_COND_NOT_WP: diff --git a/block/fops.c b/block/fops.c index 2ce7c6c4714e..c915ec0126d7 100644 --- a/block/fops.c +++ b/block/fops.c @@ -402,6 +402,11 @@ static ssize_t blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter) if (blkdev_dio_invalid(bdev, iocb, iter)) return -EINVAL; + /* HIPRI needs private as bio; HAS_METADATA keeps it as uio_meta */ + if ((iocb->ki_flags & IOCB_HIPRI) && + (iocb->ki_flags & IOCB_HAS_METADATA)) + return -EOPNOTSUPP; + if (iov_iter_rw(iter) == WRITE) { u16 max_write_streams = bdev_max_write_streams(bdev); diff --git a/block/mq-deadline.c b/block/mq-deadline.c index 5f643c0ce2a8..e5db1ee097c3 100644 --- a/block/mq-deadline.c +++ b/block/mq-deadline.c @@ -685,7 +685,7 @@ static void dd_insert_requests(struct blk_mq_hw_ctx *hctx, blk_mq_free_requests(&free); } -/* Callback from inside blk_mq_rq_ctx_init(). */ +/* Callback from inside blk_mq_rq_late_init(). */ static void dd_prepare_request(struct request *rq) { rq->elv.priv[0] = NULL; diff --git a/crypto/aes.c b/crypto/aes.c index 94791f481e98..51eee78396ea 100644 --- a/crypto/aes.c +++ b/crypto/aes.c @@ -637,7 +637,18 @@ static struct skcipher_alg skcipher_algs[] = { .decrypt = crypto_aes_cbc_decrypt, }, #endif -#if IS_ENABLED(CONFIG_CRYPTO_CTS) + /* + * Don't register library-based "cts(cbc(aes))" on architectures where + * it might block a "better" implementation from being instantiated via + * the "cts" template. These exclusions are temporary and will go away + * as the arch-optimized AES code is migrated into the library. + */ +#if IS_ENABLED(CONFIG_CRYPTO_CTS) && \ + !(IS_ENABLED(CONFIG_ARM) || \ + IS_ENABLED(CONFIG_ARM64) || \ + IS_ENABLED(CONFIG_PPC) || \ + IS_ENABLED(CONFIG_S390) || \ + IS_ENABLED(CONFIG_SPARC)) { .base.cra_name = "cts(cbc(aes))", .base.cra_driver_name = "cts-cbc-aes-lib", @@ -687,7 +698,13 @@ static struct skcipher_alg skcipher_algs[] = { .decrypt = crypto_aes_xctr_crypt, }, #endif -#if IS_ENABLED(CONFIG_CRYPTO_XTS) + /* + * Don't register library-based "xts(aes)" on architectures where it + * might block a "better" implementation from being instantiated via the + * "xts" template. This exclusion is temporary and will go away when + * the library AES-XTS is optimized for SPARC. + */ +#if IS_ENABLED(CONFIG_CRYPTO_XTS) && !IS_ENABLED(CONFIG_SPARC) { .base.cra_name = "xts(aes)", .base.cra_driver_name = "xts-aes-lib", @@ -980,7 +997,20 @@ static __maybe_unused int crypto_aes_ccm_decrypt(struct aead_request *req) } static struct aead_alg aead_algs[] = { -#if IS_ENABLED(CONFIG_CRYPTO_GCM) + /* + * Don't register library-based "gcm(aes)" and "rfc4106(gcm(aes))" on + * architectures where they might block a "better" implementation from + * being instantiated via the "gcm" and "rfc4106" templates. These + * exclusions are temporary and will go away as the arch-optimized AES + * code is migrated into the library. + */ +#if IS_ENABLED(CONFIG_CRYPTO_GCM) && \ + !(IS_ENABLED(CONFIG_ARM) || \ + IS_ENABLED(CONFIG_ARM64) || \ + IS_ENABLED(CONFIG_PPC) || \ + IS_ENABLED(CONFIG_RISCV) || \ + IS_ENABLED(CONFIG_S390) || \ + IS_ENABLED(CONFIG_SPARC)) { .base.cra_name = "gcm(aes)", .base.cra_driver_name = "gcm-aes-lib", @@ -1012,7 +1042,20 @@ static struct aead_alg aead_algs[] = { .chunksize = AES_BLOCK_SIZE, }, #endif /* CONFIG_CRYPTO_GCM */ -#if IS_ENABLED(CONFIG_CRYPTO_CCM) + /* + * Don't register library-based "ccm(aes)" on architectures where it + * might block a "better" implementation from being instantiated via the + * "ccm" template. These exclusions are temporary and will go away as + * the arch-optimized AES code is migrated into the library. + */ +#if IS_ENABLED(CONFIG_CRYPTO_CCM) && \ + !(IS_ENABLED(CONFIG_ARM) || \ + IS_ENABLED(CONFIG_ARM64) || \ + IS_ENABLED(CONFIG_PPC) || \ + IS_ENABLED(CONFIG_RISCV) || \ + IS_ENABLED(CONFIG_S390) || \ + IS_ENABLED(CONFIG_SPARC) || \ + IS_ENABLED(CONFIG_X86)) { .base.cra_name = "ccm(aes)", .base.cra_driver_name = "ccm-aes-lib", diff --git a/drivers/accel/ivpu/ivpu_drv.c b/drivers/accel/ivpu/ivpu_drv.c index 0f215392d42d..647684219e9d 100644 --- a/drivers/accel/ivpu/ivpu_drv.c +++ b/drivers/accel/ivpu/ivpu_drv.c @@ -528,6 +528,7 @@ void ivpu_prepare_for_reset(struct ivpu_device *vdev) { ivpu_hw_irq_disable(vdev); disable_irq(vdev->irq); + atomic_set(&vdev->job_timeout_detected, 0); flush_work(&vdev->irq_dct_work); flush_work(&vdev->context_abort_work); flush_work(&vdev->job_destroy_work); @@ -723,7 +724,7 @@ static int ivpu_dev_init(struct ivpu_device *vdev) vdev->context_xa_limit.max = IVPU_USER_CONTEXT_MAX_SSID; atomic64_set(&vdev->unique_id_counter, 0); atomic_set(&vdev->job_timeout_counter, 0); - atomic_set(&vdev->faults_detected, 0); + atomic_set(&vdev->job_timeout_detected, 0); xa_init_flags(&vdev->context_xa, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ); xa_init_flags(&vdev->submitted_jobs_xa, XA_FLAGS_ALLOC1); xa_init_flags(&vdev->db_xa, XA_FLAGS_ALLOC1); diff --git a/drivers/accel/ivpu/ivpu_drv.h b/drivers/accel/ivpu/ivpu_drv.h index 3b26a64ed04f..8762b8969d96 100644 --- a/drivers/accel/ivpu/ivpu_drv.h +++ b/drivers/accel/ivpu/ivpu_drv.h @@ -171,7 +171,7 @@ struct ivpu_device { struct xarray submitted_jobs_xa; struct ivpu_ipc_consumer job_done_consumer; atomic_t job_timeout_counter; - atomic_t faults_detected; + atomic_t job_timeout_detected; atomic64_t unique_id_counter; diff --git a/drivers/accel/ivpu/ivpu_job.c b/drivers/accel/ivpu/ivpu_job.c index bfdee29cd91c..b9b6efea43e8 100644 --- a/drivers/accel/ivpu/ivpu_job.c +++ b/drivers/accel/ivpu/ivpu_job.c @@ -624,7 +624,6 @@ bool ivpu_job_handle_engine_error(struct ivpu_device *vdev, u32 job_id, u32 job_ * status and ensure both are handled in the same way */ job->file_priv->has_mmu_faults = true; - atomic_set(&vdev->faults_detected, 1); queue_work(system_percpu_wq, &vdev->context_abort_work); return true; } @@ -1258,10 +1257,10 @@ static int reset_engine_and_mark_faulty_contexts(struct ivpu_device *vdev) return ret; /* - * If faults are detected, ignore guilty contexts from engine reset as NPU may not be stuck - * and could return currently running good context and faulty contexts are already marked + * If job timeout is detected, read guilty context from engine reset, for other reasons + * faulty context is already known */ - if (atomic_cmpxchg(&vdev->faults_detected, 1, 0) == 1) + if (atomic_cmpxchg(&vdev->job_timeout_detected, 1, 0) == 0) return 0; num_impacted_contexts = resp.payload.engine_reset_done.num_impacted_contexts; diff --git a/drivers/accel/ivpu/ivpu_mmu.c b/drivers/accel/ivpu/ivpu_mmu.c index 1908375ff128..c846499d96ce 100644 --- a/drivers/accel/ivpu/ivpu_mmu.c +++ b/drivers/accel/ivpu/ivpu_mmu.c @@ -960,7 +960,6 @@ void ivpu_mmu_irq_evtq_handler(struct ivpu_device *vdev) file_priv = xa_load(&vdev->context_xa, ssid); if (file_priv) { if (!READ_ONCE(file_priv->has_mmu_faults)) { - atomic_set(&vdev->faults_detected, 1); ivpu_mmu_dump_event(vdev, event); WRITE_ONCE(file_priv->has_mmu_faults, true); } diff --git a/drivers/accel/ivpu/ivpu_pm.c b/drivers/accel/ivpu/ivpu_pm.c index c1ce8329790e..de0becbfdffb 100644 --- a/drivers/accel/ivpu/ivpu_pm.c +++ b/drivers/accel/ivpu/ivpu_pm.c @@ -229,6 +229,7 @@ static void ivpu_job_timeout_work(struct work_struct *work) ivpu_jsm_state_dump(vdev); ivpu_dev_coredump(vdev); + atomic_set(&vdev->job_timeout_detected, 1); queue_work(system_percpu_wq, &vdev->context_abort_work); } diff --git a/drivers/android/binder.c b/drivers/android/binder.c index 8f2ef1bd539f..bc8bc9ee43a5 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -2930,8 +2930,9 @@ static int binder_proc_transaction(struct binder_transaction *t, t_outdated->buffer = NULL; buffer->transaction = NULL; trace_binder_transaction_update_buffer_release(buffer); - binder_release_entire_buffer(proc, NULL, buffer, false); + binder_release_entire_buffer(proc, NULL, buffer, true); binder_alloc_free_buf(&proc->alloc, buffer); + binder_free_txn_fixups(t_outdated); kfree(t_outdated); binder_stats_deleted(BINDER_STAT_TRANSACTION); } diff --git a/drivers/android/binder/node.rs b/drivers/android/binder/node.rs index 0a82af14cda3..8dc3e3f2b834 100644 --- a/drivers/android/binder/node.rs +++ b/drivers/android/binder/node.rs @@ -51,9 +51,9 @@ /// about to drop the weak reference, then the strong increment could be processed after the /// other thread has already exited, which would be too late. /// -/// Note that trying to create a `ListArc` to the node can succeed even if `has_normal_push` is +/// Note that trying to create a `ListArc` to the node can succeed even if `has_pushed_node` is /// set. This is because another thread might just have popped the node from a todo list, but not -/// yet called `do_work`. However, if `has_normal_push` is false, then creating a `ListArc` should +/// yet called `do_work`. However, if `has_pushed_node` is false, then creating a `ListArc` should /// always succeed. /// /// Like the other fields in `NodeInner`, the delivery state is protected by the process lock. @@ -738,7 +738,21 @@ fn do_work( self.do_work_locked(writer, owner_inner) } - fn cancel(self: DArc) {} + fn cancel(self: DArc) { + let _drop_outside_lock; + let mut owner_inner = self.owner.inner.lock(); + + // We only do something on BINDER_THREAD_EXIT, not process exit. + if owner_inner.is_dead { + return; + } + + // If BINDER_THREAD_EXIT is invoked on a thread with a pending node refcount update, we + // should move ourselves to ensure the refcount update is still delivered. + if let Some(node) = ListArc::try_from_arc_borrow(self.as_arc_borrow()) { + _drop_outside_lock = owner_inner.push_work(&self.owner, node); + } + } fn should_sync_wakeup(&self) -> bool { false diff --git a/drivers/android/binder/node/wrapper.rs b/drivers/android/binder/node/wrapper.rs index 6e4ca01c941a..886626ca0d42 100644 --- a/drivers/android/binder/node/wrapper.rs +++ b/drivers/android/binder/node/wrapper.rs @@ -57,7 +57,39 @@ fn do_work( node.do_work_locked(writer, owner_inner) } - fn cancel(self: DArc) {} + fn cancel(self: DArc) { + let _drop_outside_lock; + let node = &self.node; + let mut owner_inner = node.owner.inner.lock(); + + // We only do something on BINDER_THREAD_EXIT, not process exit. + if owner_inner.is_dead { + return; + } + + // We transfer the responsibility of the node refcount update to the scheduled Node because + // NodeWrapper has no way to re-create the ListArc. + let inner = node.inner.access_mut(&mut owner_inner); + + let ds = &mut inner.delivery_state; + assert!(ds.has_pushed_wrapper); + assert!(ds.has_strong_zero2one); + ds.has_pushed_wrapper = false; + + // We are changing the state to one where the Node is the strong zero2one update instead of + // the wrapper. + ds.has_weak_zero2one = false; + + if !ds.has_pushed_node { + if let Some(node2) = ListArc::try_from_arc_borrow(node.as_arc_borrow()) { + ds.has_pushed_node = true; + _drop_outside_lock = owner_inner.push_work(&node.owner, node2); + } else { + // This can't actually happen. + ds.has_strong_zero2one = false; + } + } + } fn should_sync_wakeup(&self) -> bool { false diff --git a/drivers/android/binder/rust_binderfs.c b/drivers/android/binder/rust_binderfs.c index 300cc65562d1..46a37f1858dd 100644 --- a/drivers/android/binder/rust_binderfs.c +++ b/drivers/android/binder/rust_binderfs.c @@ -72,6 +72,7 @@ struct binder_features { bool oneway_spam_detection; bool extended_error; bool freeze_notification; + bool transaction_report; }; static const struct constant_table binderfs_param_stats[] = { @@ -89,6 +90,7 @@ static struct binder_features binder_features = { .oneway_spam_detection = true, .extended_error = true, .freeze_notification = true, + .transaction_report = true, }; static inline struct binderfs_info *BINDERFS_SB(const struct super_block *sb) @@ -566,6 +568,12 @@ static int init_binder_features(struct super_block *sb) if (IS_ERR(dentry)) return PTR_ERR(dentry); + dentry = rust_binderfs_create_file(dir, "transaction_report", + &binder_features_fops, + &binder_features.transaction_report); + if (IS_ERR(dentry)) + return PTR_ERR(dentry); + return 0; } diff --git a/drivers/android/binder/thread.rs b/drivers/android/binder/thread.rs index 18a14aa8a835..24f7b5be1b0e 100644 --- a/drivers/android/binder/thread.rs +++ b/drivers/android/binder/thread.rs @@ -686,6 +686,12 @@ pub(crate) fn push_return_work(&self, reply: u32) { self.inner.lock().push_return_work(reply); } + pub(crate) fn pop_work_even_if_deferred(&self) -> Option> { + let mut thread_inner = self.inner.lock(); + thread_inner.process_work_list = true; + thread_inner.pop_work() + } + fn translate_object( &self, obj_index: usize, @@ -1678,7 +1684,7 @@ pub(crate) fn release(self: &Arc) { self.unwind_transaction_stack(); // Cancel all pending work items. - while let Ok(Some(work)) = self.get_work_local(false) { + while let Some(work) = self.pop_work_even_if_deferred() { work.into_arc().cancel(); } } diff --git a/drivers/android/binderfs.c b/drivers/android/binderfs.c index 361d69f756f5..1ca31b9a0583 100644 --- a/drivers/android/binderfs.c +++ b/drivers/android/binderfs.c @@ -189,12 +189,11 @@ static int binderfs_binder_device_create(struct inode *ref_inode, goto err; } inode->i_private = device; + binder_add_device(device); d_make_persistent(dentry, inode); fsnotify_create(root->d_inode, dentry); simple_done_creating(dentry); - binder_add_device(device); - return 0; err: diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index f482c0a6d7e9..92233fb25051 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -2972,9 +2972,10 @@ static void ata_dev_config_lpm(struct ata_device *dev) (dev->id[ATA_ID_SATA_CAPABILITY] & 0xe) == 0x2) dev->quirks |= ATA_QUIRK_NOLPM; - /* ATI specific quirk */ - if ((dev->quirks & ATA_QUIRK_NO_LPM_ON_ATI) && - ata_dev_check_adapter(dev, PCI_VENDOR_ID_ATI)) + /* ATI and AMD specific quirk */ + if ((dev->quirks & ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD) && + (ata_dev_check_adapter(dev, PCI_VENDOR_ID_ATI) || + ata_dev_check_adapter(dev, PCI_VENDOR_ID_AMD))) dev->quirks |= ATA_QUIRK_NOLPM; } @@ -4136,7 +4137,7 @@ static const char * const ata_quirk_names[] = { [__ATA_QUIRK_MAX_SEC] = "maxsec", [__ATA_QUIRK_MAX_TRIM_128M] = "maxtrim128m", [__ATA_QUIRK_NO_NCQ_ON_ATI] = "noncqonati", - [__ATA_QUIRK_NO_LPM_ON_ATI] = "nolpmonati", + [__ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD] = "nolpmonatiandamd", [__ATA_QUIRK_NO_ID_DEV_LOG] = "noiddevlog", [__ATA_QUIRK_NO_LOG_DIR] = "nologdir", [__ATA_QUIRK_NO_FUA] = "nofua", @@ -4420,15 +4421,15 @@ static const struct ata_dev_quirks_entry __ata_dev_quirks[] = { { "Samsung SSD 860*", NULL, ATA_QUIRK_NO_NCQ_TRIM | ATA_QUIRK_ZERO_AFTER_TRIM | ATA_QUIRK_NO_NCQ_ON_ATI | - ATA_QUIRK_NO_LPM_ON_ATI }, + ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD }, { "Samsung SSD 870*", NULL, ATA_QUIRK_NO_NCQ_TRIM | ATA_QUIRK_ZERO_AFTER_TRIM | ATA_QUIRK_NO_NCQ_ON_ATI | - ATA_QUIRK_NO_LPM_ON_ATI }, + ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD }, { "SAMSUNG*MZ7LH*", NULL, ATA_QUIRK_NO_NCQ_TRIM | ATA_QUIRK_ZERO_AFTER_TRIM | ATA_QUIRK_NO_NCQ_ON_ATI | - ATA_QUIRK_NO_LPM_ON_ATI }, + ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD }, { "FCCT*M500*", NULL, ATA_QUIRK_NO_NCQ_TRIM | ATA_QUIRK_ZERO_AFTER_TRIM }, diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c index 7e22bbc38238..8f9aa97a519d 100644 --- a/drivers/ata/libata-scsi.c +++ b/drivers/ata/libata-scsi.c @@ -261,12 +261,18 @@ static void ata_scsi_set_passthru_sense_fields(struct ata_queued_cmd *qc) /* descriptor format */ len = sb[7]; - desc = (char *)scsi_sense_desc_find(sb, len + 8, 9); + desc = (char *)scsi_sense_desc_find(sb, SCSI_SENSE_BUFFERSIZE, 9); if (!desc) { - if (SCSI_SENSE_BUFFERSIZE < len + 14) + /* + * The descriptor is written at sb[8 + len] and is 14 + * bytes long, so it needs len + 22 bytes of buffer. + */ + if (len + 22 > SCSI_SENSE_BUFFERSIZE) return; sb[7] = len + 14; desc = sb + 8 + len; + } else if (desc - sb > SCSI_SENSE_BUFFERSIZE - 14) { + return; } desc[0] = 9; desc[1] = 12; diff --git a/drivers/base/cacheinfo.c b/drivers/base/cacheinfo.c index 9f9c72727a05..7a47a392568a 100644 --- a/drivers/base/cacheinfo.c +++ b/drivers/base/cacheinfo.c @@ -1040,9 +1040,10 @@ static int cacheinfo_cpu_online(unsigned int cpu) rc = cache_add_dev(cpu); if (rc) goto err; - if (cpu_map_shared_cache(true, cpu, &cpu_map)) + if (cpu_map_shared_cache(true, cpu, &cpu_map)) { update_per_cpu_data_slice_size(true, cpu, cpu_map); - sched_update_llc_bytes(cpu); + sched_update_llc_bytes(cpu_map); + } return 0; err: free_cache_attributes(cpu); @@ -1059,10 +1060,10 @@ static int cacheinfo_cpu_pre_down(unsigned int cpu) cpu_cache_sysfs_exit(cpu); free_cache_attributes(cpu); - if (nr_shared > 1) + if (nr_shared > 1) { update_per_cpu_data_slice_size(false, cpu, cpu_map); - - sched_update_llc_bytes(cpu); + sched_update_llc_bytes(cpu_map); + } return 0; } diff --git a/drivers/block/drbd/drbd_nl_gen.c b/drivers/block/drbd/drbd_nl_gen.c index 9753dc789bde..a3b24aa58297 100644 --- a/drivers/block/drbd/drbd_nl_gen.c +++ b/drivers/block/drbd/drbd_nl_gen.c @@ -638,10 +638,6 @@ const struct genl_split_ops drbd_nl_ops[32] = { }, }; -static const struct genl_multicast_group drbd_nl_mcgrps[] = { - [DRBD_NLGRP_EVENTS] = { "events", }, -}; - static int __drbd_cfg_context_from_attrs(struct drbd_cfg_context *s, struct nlattr ***ret_nested_attribute_table, struct genl_info *info) diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 32bf3ba07a9d..c8bfbf8435ac 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -743,6 +743,7 @@ static int virtblk_read_zoned_limits(struct virtio_blk *vblk, dev_warn(&vdev->dev, "zero write granularity reported\n"); return -ENODEV; } + lim->zone_write_granularity = wg; lim->physical_block_size = wg; lim->io_min = wg; diff --git a/drivers/bluetooth/btintel.c b/drivers/bluetooth/btintel.c index 909a265fd906..c69a5d226e88 100644 --- a/drivers/bluetooth/btintel.c +++ b/drivers/bluetooth/btintel.c @@ -289,7 +289,8 @@ void btintel_hw_error(struct hci_dev *hdev, u8 code) goto unlock; } - bt_dev_err(hdev, "Exception info %s", (char *)(skb->data + 1)); + bt_dev_err(hdev, "Exception info %.*s", (int)(skb->len - 1), + (char *)(skb->data + 1)); kfree_skb(skb); @@ -408,8 +409,16 @@ int btintel_load_ddc_config(struct hci_dev *hdev, const char *ddc_name) /* DDC file contains one or more DDC structure which has * Length (1 byte), DDC ID (2 bytes), and DDC value (Length - 2). */ - while (fw->size > fw_ptr - fw->data) { - u8 cmd_plen = fw_ptr[0] + sizeof(u8); + while (fw->size > (size_t)(fw_ptr - fw->data)) { + size_t remaining = fw->size - (fw_ptr - fw->data); + unsigned int cmd_plen = fw_ptr[0] + 1U; + + if (cmd_plen < 3 || cmd_plen > U8_MAX || cmd_plen > remaining) { + bt_dev_err(hdev, "Malformed DDC record (plen=%u, remaining=%zu)", + cmd_plen, remaining); + release_firmware(fw); + return -EINVAL; + } skb = __hci_cmd_sync(hdev, 0xfc8b, cmd_plen, fw_ptr, HCI_INIT_TIMEOUT); diff --git a/drivers/bluetooth/btintel_pcie.c b/drivers/bluetooth/btintel_pcie.c index 6e6e2b19815c..f333b01f7465 100644 --- a/drivers/bluetooth/btintel_pcie.c +++ b/drivers/bluetooth/btintel_pcie.c @@ -1099,6 +1099,11 @@ static void btintel_pcie_msix_tx_handle(struct btintel_pcie_data *data) txq = &data->txq; + if (cr_hia >= txq->count) { + bt_dev_err(data->hdev, "TXQ: invalid cr_hia %u", cr_hia); + return; + } + while (cr_tia != cr_hia) { data->tx_wait_done = true; wake_up(&data->tx_wait_q); @@ -1192,7 +1197,7 @@ static int btintel_pcie_recv_frame(struct btintel_pcie_data *data, { int ret; u8 pkt_type; - u16 plen; + u32 plen; u32 pcie_pkt_type; void *pdata; struct hci_dev *hdev = data->hdev; @@ -1650,6 +1655,11 @@ static void btintel_pcie_msix_rx_handle(struct btintel_pcie_data *data) rxq = &data->rxq; + if (cr_hia >= rxq->count) { + bt_dev_err(hdev, "RXQ: invalid cr_hia %u", cr_hia); + return; + } + /* The firmware sends multiple CD in a single MSI-X and it needs to * process all received CDs in this interrupt. */ @@ -1657,6 +1667,12 @@ static void btintel_pcie_msix_rx_handle(struct btintel_pcie_data *data) urbd1 = &rxq->urbd1s[cr_tia]; ipc_print_urbd1(data->hdev, urbd1, cr_tia); + if (urbd1->frbd_tag >= rxq->count) { + bt_dev_err(hdev, "RXQ: invalid frbd_tag %u", + urbd1->frbd_tag); + return; + } + buf = &rxq->bufs[urbd1->frbd_tag]; if (!buf) { bt_dev_err(hdev, "RXQ: failed to get the DMA buffer for %d", @@ -2203,6 +2219,16 @@ static int btintel_pcie_send_frame(struct hci_dev *hdev, if (test_bit(BTINTEL_PCIE_RECOVERY_IN_PROGRESS, &data->flags)) return -ENODEV; + /* Account for the 4-byte PCIe type header prepended before the + * DMA copy. Written as a subtraction to avoid wrap-around on + * attacker-controlled skb->len. + */ + if (skb->len > BTINTEL_PCIE_BUFFER_SIZE - BTINTEL_PCIE_HCI_TYPE_LEN) { + bt_dev_err(hdev, "Packet too large: %u > %u", skb->len, + BTINTEL_PCIE_BUFFER_SIZE - BTINTEL_PCIE_HCI_TYPE_LEN); + return -EMSGSIZE; + } + /* Due to the fw limitation, the type header of the packet should be * 4 bytes unlike 1 byte for UART. In UART, the firmware can read * the first byte to get the packet type and redirect the rest of data diff --git a/drivers/bluetooth/btnxpuart.c b/drivers/bluetooth/btnxpuart.c index 25e7b41b349f..4e23d71d00e8 100644 --- a/drivers/bluetooth/btnxpuart.c +++ b/drivers/bluetooth/btnxpuart.c @@ -1388,9 +1388,11 @@ static int nxp_process_fw_dump(struct hci_dev *hdev, struct sk_buff *skb) msecs_to_jiffies(20000)); } - err = hci_devcd_append(hdev, skb_clone(skb, GFP_ATOMIC)); - if (err < 0) - goto free_skb; + if (IS_ENABLED(CONFIG_DEV_COREDUMP)) { + err = hci_devcd_append(hdev, skb_clone(skb, GFP_ATOMIC)); + if (err < 0) + goto free_skb; + } if (buf_len == 0) { bt_dev_warn(hdev, "==== FW dump complete ==="); diff --git a/drivers/char/tpm/tpm-interface.c b/drivers/char/tpm/tpm-interface.c index f745a098908b..1ccdbde98b69 100644 --- a/drivers/char/tpm/tpm-interface.c +++ b/drivers/char/tpm/tpm-interface.c @@ -268,7 +268,7 @@ ssize_t tpm_transmit_cmd(struct tpm_chip *chip, struct tpm_buf *buf, int err; ssize_t len; - len = tpm_transmit(chip, buf->data, PAGE_SIZE); + len = tpm_transmit(chip, buf->data, buf->capacity); if (len < 0) return len; diff --git a/drivers/char/tpm/tpm2-cmd.c b/drivers/char/tpm/tpm2-cmd.c index 48cec39995fe..ae22295df798 100644 --- a/drivers/char/tpm/tpm2-cmd.c +++ b/drivers/char/tpm/tpm2-cmd.c @@ -281,8 +281,10 @@ int tpm2_get_random(struct tpm_chip *chip, u8 *dest, size_t max) } tpm_buf_append_u16(buf, num_bytes); err = tpm_buf_fill_hmac_session(chip, buf); - if (err) + if (err) { + tpm2_end_auth_session(chip); return err; + } err = tpm_transmit_cmd(chip, buf, offsetof(struct tpm2_get_random_out, @@ -305,7 +307,7 @@ int tpm2_get_random(struct tpm_chip *chip, u8 *dest, size_t max) out = (struct tpm2_get_random_out *)&buf->data[offset]; recd = min_t(u32, be16_to_cpu(out->size), num_bytes); if (tpm_buf_length(buf) < - TPM_HEADER_SIZE + + offset + offsetof(struct tpm2_get_random_out, buffer) + recd) { tpm2_end_auth_session(chip); diff --git a/drivers/char/tpm/tpm2-sessions.c b/drivers/char/tpm/tpm2-sessions.c index cf8f1fd6790b..ca1e2bf424e1 100644 --- a/drivers/char/tpm/tpm2-sessions.c +++ b/drivers/char/tpm/tpm2-sessions.c @@ -975,6 +975,7 @@ static int tpm2_load_null(struct tpm_chip *chip, u32 *null_key) /* Deduce from the name change TPM interference: */ dev_err(&chip->dev, "null key integrity check failed\n"); tpm2_flush_context(chip, tmp_null_key); + rc = -ENODEV; err: if (rc) { diff --git a/drivers/clk/spacemit/ccu-k3.c b/drivers/clk/spacemit/ccu-k3.c index 92b930d5ff30..39df15c8a3e6 100644 --- a/drivers/clk/spacemit/ccu-k3.c +++ b/drivers/clk/spacemit/ccu-k3.c @@ -30,14 +30,87 @@ static const struct ccu_pll_rate_tbl pll2_rate_tbl[] = { }; static const struct ccu_pll_rate_tbl pll3_rate_tbl[] = { + CCU_PLLA_RATE(1050000000UL, 0x0b2b3000, 0x00000000, 0xa0458080), + CCU_PLLA_RATE(1100000000UL, 0x0b2d3555, 0x00005500, 0xa0458181), + CCU_PLLA_RATE(1150000000UL, 0x0b2f3aaa, 0x0000ab00, 0xa0458181), + CCU_PLLA_RATE(1200000000UL, 0x0b320000, 0x00000000, 0xa0458181), + CCU_PLLA_RATE(1250000000UL, 0x0b340555, 0x00005500, 0xa0458181), + CCU_PLLA_RATE(1300000000UL, 0x0b360aaa, 0x0000ab00, 0xa0458282), + CCU_PLLA_RATE(1350000000UL, 0x0b381000, 0x00000000, 0xa0458282), + CCU_PLLA_RATE(1400000000UL, 0x0b3a1555, 0x00005500, 0xa0458282), + CCU_PLLA_RATE(1450000000UL, 0x0b3c1aaa, 0x0000ab00, 0xa0458383), + CCU_PLLA_RATE(1500000000UL, 0x0b3e2000, 0x00000000, 0xa0458383), + CCU_PLLA_RATE(1550000000UL, 0x0b402555, 0x00005500, 0xa0458383), + CCU_PLLA_RATE(1600000000UL, 0x0b422aaa, 0x0000ab00, 0xa0458484), + CCU_PLLA_RATE(1650000000UL, 0x0b443000, 0x00000000, 0xa0458484), + CCU_PLLA_RATE(1700000000UL, 0x0b463555, 0x00005500, 0xa0458484), + CCU_PLLA_RATE(1750000000UL, 0x0b483aaa, 0x0000ab00, 0xa0458585), + CCU_PLLA_RATE(1800000000UL, 0x0b4b0000, 0x00000000, 0xa0458585), + CCU_PLLA_RATE(1850000000UL, 0x0b4d0555, 0x00005500, 0xa0458585), + CCU_PLLA_RATE(1900000000UL, 0x0b4f0aaa, 0x0000ab00, 0xa0458686), + CCU_PLLA_RATE(1950000000UL, 0x0b511000, 0x00000000, 0xa0458686), + CCU_PLLA_RATE(2000000000UL, 0x0b292aaa, 0x0000ab00, 0xa0558686), + CCU_PLLA_RATE(2050000000UL, 0x0b2a2d55, 0x00005500, 0xa0558686), + CCU_PLLA_RATE(2100000000UL, 0x0b2b3000, 0x00000000, 0xa0558787), + CCU_PLLA_RATE(2150000000UL, 0x0b2c32aa, 0x0000ab00, 0xa0558787), CCU_PLLA_RATE(2200000000UL, 0x0b2d3555, 0x00005500, 0xa0558787), + CCU_PLLA_RATE(2250000000UL, 0x0b2e3800, 0x00000000, 0xa0558888), + CCU_PLLA_RATE(2300000000UL, 0x0b2f3aaa, 0x0000ab00, 0xa0558888), + CCU_PLLA_RATE(2350000000UL, 0x0b303d55, 0x00005500, 0xa0558888), + CCU_PLLA_RATE(2400000000UL, 0x0b320000, 0x00000000, 0xa0558989), }; static const struct ccu_pll_rate_tbl pll4_rate_tbl[] = { + CCU_PLLA_RATE(1050000000UL, 0x0b2b3000, 0x00000000, 0xa0458080), + CCU_PLLA_RATE(1100000000UL, 0x0b2d3555, 0x00005500, 0xa0458181), + CCU_PLLA_RATE(1150000000UL, 0x0b2f3aaa, 0x0000ab00, 0xa0458181), + CCU_PLLA_RATE(1200000000UL, 0x0b320000, 0x00000000, 0xa0458181), + CCU_PLLA_RATE(1250000000UL, 0x0b340555, 0x00005500, 0xa0458181), + CCU_PLLA_RATE(1300000000UL, 0x0b360aaa, 0x0000ab00, 0xa0458282), + CCU_PLLA_RATE(1350000000UL, 0x0b381000, 0x00000000, 0xa0458282), + CCU_PLLA_RATE(1400000000UL, 0x0b3a1555, 0x00005500, 0xa0458282), + CCU_PLLA_RATE(1450000000UL, 0x0b3c1aaa, 0x0000ab00, 0xa0458383), + CCU_PLLA_RATE(1500000000UL, 0x0b3e2000, 0x00000000, 0xa0458383), + CCU_PLLA_RATE(1550000000UL, 0x0b402555, 0x00005500, 0xa0458383), + CCU_PLLA_RATE(1600000000UL, 0x0b422aaa, 0x0000ab00, 0xa0458484), + CCU_PLLA_RATE(1650000000UL, 0x0b443000, 0x00000000, 0xa0458484), + CCU_PLLA_RATE(1700000000UL, 0x0b463555, 0x00005500, 0xa0458484), + CCU_PLLA_RATE(1750000000UL, 0x0b483aaa, 0x0000ab00, 0xa0458585), + CCU_PLLA_RATE(1800000000UL, 0x0b4b0000, 0x00000000, 0xa0458585), + CCU_PLLA_RATE(1850000000UL, 0x0b4d0555, 0x00005500, 0xa0458585), + CCU_PLLA_RATE(1900000000UL, 0x0b4f0aaa, 0x0000ab00, 0xa0458686), + CCU_PLLA_RATE(1950000000UL, 0x0b511000, 0x00000000, 0xa0458686), + CCU_PLLA_RATE(2000000000UL, 0x0b292aaa, 0x0000ab00, 0xa0558686), + CCU_PLLA_RATE(2050000000UL, 0x0b2a2d55, 0x00005500, 0xa0558686), + CCU_PLLA_RATE(2100000000UL, 0x0b2b3000, 0x00000000, 0xa0558787), + CCU_PLLA_RATE(2150000000UL, 0x0b2c32aa, 0x0000ab00, 0xa0558787), CCU_PLLA_RATE(2200000000UL, 0x0b2d3555, 0x00005500, 0xa0558787), + CCU_PLLA_RATE(2250000000UL, 0x0b2e3800, 0x00000000, 0xa0558888), + CCU_PLLA_RATE(2300000000UL, 0x0b2f3aaa, 0x0000ab00, 0xa0558888), + CCU_PLLA_RATE(2350000000UL, 0x0b303d55, 0x00005500, 0xa0558888), + CCU_PLLA_RATE(2400000000UL, 0x0b320000, 0x00000000, 0xa0558989), }; static const struct ccu_pll_rate_tbl pll5_rate_tbl[] = { + CCU_PLLA_RATE(1050000000UL, 0x0b2b3000, 0x00000000, 0xa0458080), + CCU_PLLA_RATE(1100000000UL, 0x0b2d3555, 0x00005500, 0xa0458181), + CCU_PLLA_RATE(1150000000UL, 0x0b2f3aaa, 0x0000ab00, 0xa0458181), + CCU_PLLA_RATE(1200000000UL, 0x0b320000, 0x00000000, 0xa0458181), + CCU_PLLA_RATE(1250000000UL, 0x0b340555, 0x00005500, 0xa0458181), + CCU_PLLA_RATE(1300000000UL, 0x0b360aaa, 0x0000ab00, 0xa0458282), + CCU_PLLA_RATE(1350000000UL, 0x0b381000, 0x00000000, 0xa0458282), + CCU_PLLA_RATE(1400000000UL, 0x0b3a1555, 0x00005500, 0xa0458282), + CCU_PLLA_RATE(1450000000UL, 0x0b3c1aaa, 0x0000ab00, 0xa0458383), + CCU_PLLA_RATE(1500000000UL, 0x0b3e2000, 0x00000000, 0xa0458383), + CCU_PLLA_RATE(1550000000UL, 0x0b402555, 0x00005500, 0xa0458383), + CCU_PLLA_RATE(1600000000UL, 0x0b422aaa, 0x0000ab00, 0xa0458484), + CCU_PLLA_RATE(1650000000UL, 0x0b443000, 0x00000000, 0xa0458484), + CCU_PLLA_RATE(1700000000UL, 0x0b463555, 0x00005500, 0xa0458484), + CCU_PLLA_RATE(1750000000UL, 0x0b483aaa, 0x0000ab00, 0xa0458585), + CCU_PLLA_RATE(1800000000UL, 0x0b4b0000, 0x00000000, 0xa0458585), + CCU_PLLA_RATE(1850000000UL, 0x0b4d0555, 0x00005500, 0xa0458585), + CCU_PLLA_RATE(1900000000UL, 0x0b4f0aaa, 0x0000ab00, 0xa0458686), + CCU_PLLA_RATE(1950000000UL, 0x0b511000, 0x00000000, 0xa0458686), CCU_PLLA_RATE(2000000000UL, 0x0b292aaa, 0x0000ab00, 0xa0558686), }; @@ -50,6 +123,25 @@ static const struct ccu_pll_rate_tbl pll7_rate_tbl[] = { }; static const struct ccu_pll_rate_tbl pll8_rate_tbl[] = { + CCU_PLLA_RATE(1050000000UL, 0x0b2b3000, 0x00000000, 0xa0458080), + CCU_PLLA_RATE(1100000000UL, 0x0b2d3555, 0x00005500, 0xa0458181), + CCU_PLLA_RATE(1150000000UL, 0x0b2f3aaa, 0x0000ab00, 0xa0458181), + CCU_PLLA_RATE(1200000000UL, 0x0b320000, 0x00000000, 0xa0458181), + CCU_PLLA_RATE(1250000000UL, 0x0b340555, 0x00005500, 0xa0458181), + CCU_PLLA_RATE(1300000000UL, 0x0b360aaa, 0x0000ab00, 0xa0458282), + CCU_PLLA_RATE(1350000000UL, 0x0b381000, 0x00000000, 0xa0458282), + CCU_PLLA_RATE(1400000000UL, 0x0b3a1555, 0x00005500, 0xa0458282), + CCU_PLLA_RATE(1450000000UL, 0x0b3c1aaa, 0x0000ab00, 0xa0458383), + CCU_PLLA_RATE(1500000000UL, 0x0b3e2000, 0x00000000, 0xa0458383), + CCU_PLLA_RATE(1550000000UL, 0x0b402555, 0x00005500, 0xa0458383), + CCU_PLLA_RATE(1600000000UL, 0x0b422aaa, 0x0000ab00, 0xa0458484), + CCU_PLLA_RATE(1650000000UL, 0x0b443000, 0x00000000, 0xa0458484), + CCU_PLLA_RATE(1700000000UL, 0x0b463555, 0x00005500, 0xa0458484), + CCU_PLLA_RATE(1750000000UL, 0x0b483aaa, 0x0000ab00, 0xa0458585), + CCU_PLLA_RATE(1800000000UL, 0x0b4b0000, 0x00000000, 0xa0458585), + CCU_PLLA_RATE(1850000000UL, 0x0b4d0555, 0x00005500, 0xa0458585), + CCU_PLLA_RATE(1900000000UL, 0x0b4f0aaa, 0x0000ab00, 0xa0458686), + CCU_PLLA_RATE(1950000000UL, 0x0b511000, 0x00000000, 0xa0458686), CCU_PLLA_RATE(2000000000UL, 0x0b292aaa, 0x0000ab00, 0xa0558686), }; diff --git a/drivers/clk/ti/composite.c b/drivers/clk/ti/composite.c index 83c3592cd179..c379bbdae25a 100644 --- a/drivers/clk/ti/composite.c +++ b/drivers/clk/ti/composite.c @@ -52,7 +52,7 @@ static const struct clk_ops ti_composite_gate_ops = { struct component_clk { int num_parents; - struct clk_parent_data *parent_data; + const char **parent_names; struct device_node *node; int type; struct clk_hw *hw; @@ -116,7 +116,7 @@ static void __init _register_composite(void *user, struct clk_hw_omap_comp *cclk = to_clk_hw_comp(hw); struct component_clk *comp; int num_parents = 0; - struct clk_parent_data *parent_data = NULL; + const char **parent_names = NULL; const char *name; int i; int ret; @@ -155,7 +155,7 @@ static void __init _register_composite(void *user, continue; if (comp->num_parents) { num_parents = comp->num_parents; - parent_data = comp->parent_data; + parent_names = comp->parent_names; break; } } @@ -166,8 +166,8 @@ static void __init _register_composite(void *user, } name = ti_dt_clk_name(node); - clk = clk_register_composite_pdata(NULL, name, - parent_data, num_parents, + clk = clk_register_composite(NULL, name, + parent_names, num_parents, _get_hw(cclk, CLK_COMPONENT_TYPE_MUX), &ti_clk_mux_ops, _get_hw(cclk, CLK_COMPONENT_TYPE_DIVIDER), @@ -190,7 +190,7 @@ static void __init _register_composite(void *user, if (!cclk->comp_clks[i]) continue; list_del(&cclk->comp_clks[i]->link); - kfree(cclk->comp_clks[i]->parent_data); + kfree(cclk->comp_clks[i]->parent_names); kfree(cclk->comp_clks[i]); } @@ -237,9 +237,8 @@ int __init ti_clk_add_component(struct device_node *node, struct clk_hw *hw, int type) { unsigned int num_parents; - struct clk_parent_data *parent_data; + const char **parent_names; struct component_clk *clk; - unsigned int i; num_parents = of_clk_get_parent_count(node); @@ -248,21 +247,20 @@ int __init ti_clk_add_component(struct device_node *node, struct clk_hw *hw, return -EINVAL; } - parent_data = kzalloc_objs(*parent_data, num_parents); - if (!parent_data) + parent_names = kcalloc(num_parents, sizeof(char *), GFP_KERNEL); + if (!parent_names) return -ENOMEM; - for (i = 0; i < num_parents; i++) - parent_data[i].index = i; + of_clk_parent_fill(node, parent_names, num_parents); clk = kzalloc_obj(*clk); if (!clk) { - kfree(parent_data); + kfree(parent_names); return -ENOMEM; } clk->num_parents = num_parents; - clk->parent_data = parent_data; + clk->parent_names = parent_names; clk->hw = hw; clk->node = node; clk->type = type; diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 96515880b4ac..54dde8419bdc 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -2590,8 +2590,8 @@ static void cpufreq_update_pressure(struct cpufreq_policy *policy) cpu = cpumask_first(policy->related_cpus); max_freq = arch_scale_freq_ref(cpu); - if (!max_freq) - max_freq = policy->cpuinfo.max_freq; + if (!max_freq && cpufreq_driver->scale_freq_ref) + max_freq = cpufreq_driver->scale_freq_ref(policy); capped_freq = policy->max; diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index ceb340f7a110..17158cb25dd8 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1135,6 +1135,14 @@ static bool hybrid_clear_max_perf_cpu(void) return ret; } +static unsigned int intel_pstate_scale_freq_ref(struct cpufreq_policy *policy) +{ + if (READ_ONCE(all_cpu_data[policy->cpu]->capacity_perf)) + return policy->cpuinfo.max_freq; + + return 0; +} + static void intel_pstate_update_freq_limits(struct cpudata *cpu) { int scaling = cpu->pstate.scaling; @@ -3088,6 +3096,7 @@ static struct cpufreq_driver intel_pstate = { .offline = intel_pstate_cpu_offline, .online = intel_pstate_cpu_online, .update_limits = intel_pstate_update_limits, + .scale_freq_ref = intel_pstate_scale_freq_ref, .name = "intel_pstate", }; @@ -3411,6 +3420,7 @@ static struct cpufreq_driver intel_cpufreq = { .suspend = intel_cpufreq_suspend, .resume = intel_pstate_resume, .update_limits = intel_pstate_update_limits, + .scale_freq_ref = intel_pstate_scale_freq_ref, .name = "intel_cpufreq", }; diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c index 45365214fbef..fb24fd53f2e1 100644 --- a/drivers/dpll/dpll_netlink.c +++ b/drivers/dpll/dpll_netlink.c @@ -1210,8 +1210,7 @@ dpll_pin_ref_sync_state_set(struct dpll_pin *pin, struct dpll_device *dpll; int ret; - ref_sync_pin = xa_find(&pin->ref_sync_pins, &ref_sync_pin_idx, - ULONG_MAX, XA_PRESENT); + ref_sync_pin = xa_load(&pin->ref_sync_pins, ref_sync_pin_idx); if (!ref_sync_pin) { NL_SET_ERR_MSG(extack, "reference sync pin not found"); return -EINVAL; diff --git a/drivers/edac/altera_edac.c b/drivers/edac/altera_edac.c index 68846f583eee..bb95dab847b3 100644 --- a/drivers/edac/altera_edac.c +++ b/drivers/edac/altera_edac.c @@ -453,15 +453,6 @@ static int altr_sdram_probe(struct platform_device *pdev) return res; } -static void altr_sdram_remove(struct platform_device *pdev) -{ - struct mem_ctl_info *mci = platform_get_drvdata(pdev); - - edac_mc_del_mc(&pdev->dev); - edac_mc_free(mci); - platform_set_drvdata(pdev, NULL); -} - /* * If you want to suspend, need to disable EDAC by removing it * from the device tree or defconfig. @@ -481,13 +472,13 @@ static const struct dev_pm_ops altr_sdram_pm_ops = { static struct platform_driver altr_sdram_edac_driver = { .probe = altr_sdram_probe, - .remove = altr_sdram_remove, .driver = { .name = "altr_sdram_edac", #ifdef CONFIG_PM .pm = &altr_sdram_pm_ops, #endif .of_match_table = altr_sdram_ctrl_of_match, + .suppress_bind_attrs = true, }, }; @@ -517,6 +508,7 @@ static struct platform_driver altr_edac_driver = { .driver = { .name = "socfpga_ecc_manager", .of_match_table = altr_edac_of_match, + .suppress_bind_attrs = true, }, }; module_platform_driver(altr_edac_driver); @@ -803,22 +795,12 @@ static int altr_edac_device_probe(struct platform_device *pdev) return res; } -static void altr_edac_device_remove(struct platform_device *pdev) -{ - struct edac_device_ctl_info *dci = platform_get_drvdata(pdev); - struct altr_edac_device_dev *drvdata = dci->pvt_info; - - debugfs_remove_recursive(drvdata->debugfs_dir); - edac_device_del_device(&pdev->dev); - edac_device_free_ctl_info(dci); -} - static struct platform_driver altr_edac_device_driver = { .probe = altr_edac_device_probe, - .remove = altr_edac_device_remove, .driver = { .name = "altr_edac_device", .of_match_table = altr_edac_device_of_match, + .suppress_bind_attrs = true, }, }; module_platform_driver(altr_edac_device_driver); @@ -944,7 +926,7 @@ static int __maybe_unused altr_init_memory_port(void __iomem *ioaddr, int port) return ret; } -static __init int __maybe_unused +static int __maybe_unused altr_init_a10_ecc_block(struct device_node *np, u32 irq_mask, u32 ecc_ctrl_en_mask, bool dual_port) { @@ -1019,7 +1001,7 @@ altr_init_a10_ecc_block(struct device_node *np, u32 irq_mask, static int validate_parent_available(struct device_node *np); static const struct of_device_id altr_edac_a10_device_of_match[]; -static int __init __maybe_unused altr_init_a10_ecc_device_type(char *compat) +static int __maybe_unused altr_init_a10_ecc_device_type(char *compat) { int irq; struct device_node *child, *np; @@ -1348,7 +1330,7 @@ static const struct edac_device_prv_data a10_l2ecc_data = { #ifdef CONFIG_EDAC_ALTERA_ETHERNET -static int __init socfpga_init_ethernet_ecc(struct altr_edac_device_dev *dev) +static int socfpga_init_ethernet_ecc(struct altr_edac_device_dev *dev) { int ret; @@ -1378,7 +1360,7 @@ static const struct edac_device_prv_data a10_enetecc_data = { #ifdef CONFIG_EDAC_ALTERA_NAND -static int __init socfpga_init_nand_ecc(struct altr_edac_device_dev *device) +static int socfpga_init_nand_ecc(struct altr_edac_device_dev *device) { int ret; @@ -1408,7 +1390,7 @@ static const struct edac_device_prv_data a10_nandecc_data = { #ifdef CONFIG_EDAC_ALTERA_DMA -static int __init socfpga_init_dma_ecc(struct altr_edac_device_dev *device) +static int socfpga_init_dma_ecc(struct altr_edac_device_dev *device) { int ret; @@ -1438,7 +1420,7 @@ static const struct edac_device_prv_data a10_dmaecc_data = { #ifdef CONFIG_EDAC_ALTERA_USB -static int __init socfpga_init_usb_ecc(struct altr_edac_device_dev *device) +static int socfpga_init_usb_ecc(struct altr_edac_device_dev *device) { int ret; @@ -1468,7 +1450,7 @@ static const struct edac_device_prv_data a10_usbecc_data = { #ifdef CONFIG_EDAC_ALTERA_QSPI -static int __init socfpga_init_qspi_ecc(struct altr_edac_device_dev *device) +static int socfpga_init_qspi_ecc(struct altr_edac_device_dev *device) { int ret; @@ -1524,6 +1506,7 @@ static int altr_portb_setup(struct altr_edac_device_dev *device) dci = edac_device_alloc_ctl_info(sizeof(*altdev), ecc_name, 1, ecc_name, 1, 0, edac_idx); if (!dci) { + of_node_put(np); edac_printk(KERN_ERR, EDAC_DEVICE, "%s: Unable to allocate PortB EDAC device\n", ecc_name); @@ -1534,8 +1517,11 @@ static int altr_portb_setup(struct altr_edac_device_dev *device) altdev = dci->pvt_info; *altdev = *device; - if (!devres_open_group(device->edac->dev, altr_portb_setup, GFP_KERNEL)) + if (!devres_open_group(device->edac->dev, altr_portb_setup, GFP_KERNEL)) { + edac_device_free_ctl_info(dci); + of_node_put(np); return -ENOMEM; + } /* Update PortB specific values */ altdev->edac_dev_name = ecc_name; @@ -1600,6 +1586,8 @@ static int altr_portb_setup(struct altr_edac_device_dev *device) rc = -ENOMEM; goto err_release_group_1; } + of_node_put(np); + altr_create_edacdev_dbgfs(dci, prv); list_add(&altdev->next, &altdev->edac->a10_ecc_devices); @@ -1609,14 +1597,20 @@ static int altr_portb_setup(struct altr_edac_device_dev *device) return 0; err_release_group_1: - edac_device_free_ctl_info(dci); + /* + * Release the devres group first so the managed IRQs are + * unregistered before dci (which contains the IRQ handler's + * data via dci->pvt_info) is freed, avoiding a use-after-free. + */ devres_release_group(device->edac->dev, altr_portb_setup); + edac_device_free_ctl_info(dci); + of_node_put(np); edac_printk(KERN_ERR, EDAC_DEVICE, "%s:Error setting up EDAC device: %d\n", ecc_name, rc); return rc; } -static int __init socfpga_init_sdmmc_ecc(struct altr_edac_device_dev *device) +static int socfpga_init_sdmmc_ecc(struct altr_edac_device_dev *device) { int rc = -ENODEV; struct device_node *child; @@ -1638,7 +1632,7 @@ static int __init socfpga_init_sdmmc_ecc(struct altr_edac_device_dev *device) goto exit; /* Setup portB */ - return altr_portb_setup(device); + rc = altr_portb_setup(device); exit: of_node_put(child); @@ -2013,9 +2007,17 @@ static int altr_edac_a10_device_add(struct altr_arria10_edac *edac, return 0; err_release_group1: + /* + * Release the devres group first so the managed IRQs are + * unregistered before dci (which contains the IRQ handler's + * data via dci->pvt_info) is freed, avoiding a use-after-free. + */ + devres_release_group(edac->dev, NULL); edac_device_free_ctl_info(dci); + goto err_print; err_release_group: devres_release_group(edac->dev, NULL); +err_print: edac_printk(KERN_ERR, EDAC_DEVICE, "%s:Error setting up EDAC device: %d\n", ecc_name, rc); @@ -2214,6 +2216,7 @@ static struct platform_driver altr_edac_a10_driver = { .driver = { .name = "socfpga_a10_ecc_manager", .of_match_table = altr_edac_a10_of_match, + .suppress_bind_attrs = true, }, }; module_platform_driver(altr_edac_a10_driver); diff --git a/drivers/edac/versalnet_edac.c b/drivers/edac/versalnet_edac.c index 9e65c4b1d99d..eae9fea60db1 100644 --- a/drivers/edac/versalnet_edac.c +++ b/drivers/edac/versalnet_edac.c @@ -953,6 +953,7 @@ static void mc_remove(struct platform_device *pdev) remove_versalnet(priv); rproc_shutdown(priv->mcdi->r5_rproc); cdx_mcdi_finish(priv->mcdi); + rproc_put(priv->mcdi->r5_rproc); kfree(priv->mcdi); } diff --git a/drivers/firewire/core-cdev.c b/drivers/firewire/core-cdev.c index e49d8a58be09..664952a67a11 100644 --- a/drivers/firewire/core-cdev.c +++ b/drivers/firewire/core-cdev.c @@ -1397,8 +1397,10 @@ static void iso_resource_auto_work(struct work_struct *work) } else { // Transit from allocation to reallocation, except if the client requested // deallocation in the meantime. - scoped_guard(spinlock_irq, &client->lock) - r->todo = ISO_RES_AUTO_REALLOC; + scoped_guard(spinlock_irq, &client->lock) { + if (r->todo == ISO_RES_AUTO_ALLOC) + r->todo = ISO_RES_AUTO_REALLOC; + } if (channel >= 0) r->params.channels_mask = BIT_ULL(channel); diff --git a/drivers/gpio/gpio-arizona.c b/drivers/gpio/gpio-arizona.c index a7e98d395d8e..88cf013f0d90 100644 --- a/drivers/gpio/gpio-arizona.c +++ b/drivers/gpio/gpio-arizona.c @@ -115,8 +115,12 @@ static int arizona_gpio_direction_out(struct gpio_chip *chip, if (value) value = ARIZONA_GPN_LVL; - return regmap_update_bits(arizona->regmap, ARIZONA_GPIO1_CTRL + offset, - ARIZONA_GPN_DIR | ARIZONA_GPN_LVL, value); + ret = regmap_update_bits(arizona->regmap, ARIZONA_GPIO1_CTRL + offset, + ARIZONA_GPN_DIR | ARIZONA_GPN_LVL, value); + if (ret < 0 && (val & ARIZONA_GPN_DIR) && persistent) + pm_runtime_put_autosuspend(chip->parent); + + return ret; } static int arizona_gpio_set(struct gpio_chip *chip, unsigned int offset, diff --git a/drivers/gpio/gpio-mvebu.c b/drivers/gpio/gpio-mvebu.c index 93b8a08b04b9..a2796240fe14 100644 --- a/drivers/gpio/gpio-mvebu.c +++ b/drivers/gpio/gpio-mvebu.c @@ -1034,6 +1034,8 @@ static int mvebu_gpio_suspend(struct platform_device *pdev, pm_message_t state) static int mvebu_gpio_resume(struct platform_device *pdev) { struct mvebu_gpio_chip *mvchip = platform_get_drvdata(pdev); + u32 edge_cache = ~0U, level_cache = ~0U; + unsigned long flags; int i; regmap_write(mvchip->regs, GPIO_OUT_OFF + mvchip->offset, @@ -1045,32 +1047,51 @@ static int mvebu_gpio_resume(struct platform_device *pdev) regmap_write(mvchip->regs, GPIO_IN_POL_OFF + mvchip->offset, mvchip->in_pol_reg); + /* + * genirq skips mask_irq() for a line it already considers masked, so + * unmasking one behind its back leaves an asserted level line that + * nobody masks. Restore only bits the irqchip cache still has set. + * + * Snapshot the caches under the raw spinlock, but release it before + * the regmap writes below: regmap_write() takes a sleepable lock on + * PREEMPT_RT. + */ + if (mvchip->domain) { + struct irq_chip_generic *gc; + + gc = irq_get_domain_generic_chip(mvchip->domain, 0); + raw_spin_lock_irqsave(&gc->lock, flags); + level_cache = gc->chip_types[0].mask_cache_priv; + edge_cache = gc->chip_types[1].mask_cache_priv; + raw_spin_unlock_irqrestore(&gc->lock, flags); + } + switch (mvchip->soc_variant) { case MVEBU_GPIO_SOC_VARIANT_ORION: case MVEBU_GPIO_SOC_VARIANT_A8K: regmap_write(mvchip->regs, GPIO_EDGE_MASK_OFF + mvchip->offset, - mvchip->edge_mask_regs[0]); + mvchip->edge_mask_regs[0] & edge_cache); regmap_write(mvchip->regs, GPIO_LEVEL_MASK_OFF + mvchip->offset, - mvchip->level_mask_regs[0]); + mvchip->level_mask_regs[0] & level_cache); break; case MVEBU_GPIO_SOC_VARIANT_MV78200: for (i = 0; i < 2; i++) { regmap_write(mvchip->regs, GPIO_EDGE_MASK_MV78200_OFF(i), - mvchip->edge_mask_regs[i]); + mvchip->edge_mask_regs[i] & edge_cache); regmap_write(mvchip->regs, GPIO_LEVEL_MASK_MV78200_OFF(i), - mvchip->level_mask_regs[i]); + mvchip->level_mask_regs[i] & level_cache); } break; case MVEBU_GPIO_SOC_VARIANT_ARMADAXP: for (i = 0; i < 4; i++) { regmap_write(mvchip->regs, GPIO_EDGE_MASK_ARMADAXP_OFF(i), - mvchip->edge_mask_regs[i]); + mvchip->edge_mask_regs[i] & edge_cache); regmap_write(mvchip->regs, GPIO_LEVEL_MASK_ARMADAXP_OFF(i), - mvchip->level_mask_regs[i]); + mvchip->level_mask_regs[i] & level_cache); } break; default: diff --git a/drivers/gpio/gpio-tps65219.c b/drivers/gpio/gpio-tps65219.c index 457fd8a589e8..6958466455d0 100644 --- a/drivers/gpio/gpio-tps65219.c +++ b/drivers/gpio/gpio-tps65219.c @@ -79,7 +79,7 @@ static int tps65219_gpio_get(struct gpio_chip *gc, unsigned int offset) if (ret) return ret; - ret = !!(val & BIT(TPS65219_MFP_GPIO_STATUS_MASK)); + ret = !!(val & TPS65219_MFP_GPIO_STATUS_MASK); dev_warn(dev, "GPIO%d = %d, MULTI_DEVICE_ENABLE, not a standard GPIO\n", offset, ret); /* @@ -87,7 +87,7 @@ static int tps65219_gpio_get(struct gpio_chip *gc, unsigned int offset) * status bit. */ - if (tps65219_gpio_get_direction(gc, offset) == GPIO_LINE_DIRECTION_OUT) + if (gc->get_direction(gc, offset) == GPIO_LINE_DIRECTION_OUT) return -ENOTSUPP; return ret; @@ -158,8 +158,10 @@ static int tps65214_gpio_change_direction(struct gpio_chip *gc, unsigned int off if (ret) dev_err(dev, "GPIO%d configured as VSEL, not GPIO\n", offset); + val = direction == GPIO_LINE_DIRECTION_OUT ? + TPS65214_GPIO0_DIR_MASK : 0; ret = regmap_update_bits(gpio->tps->regmap, TPS65219_REG_GENERAL_CONFIG, - TPS65214_GPIO0_DIR_MASK, direction); + TPS65214_GPIO0_DIR_MASK, val); if (ret) dev_err(dev, "Fail to change direction to %u for GPIO%d.\n", direction, offset); @@ -176,7 +178,7 @@ static int tps65219_gpio_direction_input(struct gpio_chip *gc, unsigned int offs return -ENOTSUPP; } - if (tps65219_gpio_get_direction(gc, offset) == GPIO_LINE_DIRECTION_IN) + if (gc->get_direction(gc, offset) == GPIO_LINE_DIRECTION_IN) return 0; return gpio->change_dir(gc, offset, GPIO_LINE_DIRECTION_IN); @@ -190,7 +192,7 @@ static int tps65219_gpio_direction_output(struct gpio_chip *gc, unsigned int off if (offset != TPS6521X_GPIO0_IDX) return 0; - if (tps65219_gpio_get_direction(gc, offset) == GPIO_LINE_DIRECTION_OUT) + if (gc->get_direction(gc, offset) == GPIO_LINE_DIRECTION_OUT) return 0; return gpio->change_dir(gc, offset, GPIO_LINE_DIRECTION_OUT); diff --git a/drivers/gpio/gpio-zynq.c b/drivers/gpio/gpio-zynq.c index 15a79d9a2e9e..13e4c5f0e297 100644 --- a/drivers/gpio/gpio-zynq.c +++ b/drivers/gpio/gpio-zynq.c @@ -798,15 +798,7 @@ static int zynq_gpio_runtime_resume(struct device *dev) static int zynq_gpio_request(struct gpio_chip *chip, unsigned int offset) { - int ret; - - ret = pm_runtime_get_sync(chip->parent); - - /* - * If the device is already active pm_runtime_get() will return 1 on - * success, but gpio_request still needs to return 0. - */ - return ret < 0 ? ret : 0; + return pm_runtime_resume_and_get(chip->parent); } static void zynq_gpio_free(struct gpio_chip *chip, unsigned int offset) diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c index d1105b7ae437..5d53bfcdf726 100644 --- a/drivers/gpio/gpiolib-cdev.c +++ b/drivers/gpio/gpiolib-cdev.c @@ -2172,18 +2172,19 @@ static void gpio_v2_line_info_changed_to_v1( #endif /* CONFIG_GPIO_CDEV_V1 */ -static void gpio_desc_to_lineinfo(struct gpio_desc *desc, - struct gpio_v2_line_info *info, bool atomic) +static int gpio_desc_to_lineinfo(struct gpio_desc *desc, + struct gpio_v2_line_info *info, bool atomic) { u32 debounce_period_us; unsigned long dflags; const char *label; + memset(info, 0, sizeof(*info)); + CLASS(gpio_chip_guard, guard)(desc); if (!guard.gc) - return; + return -ENODEV; - memset(info, 0, sizeof(*info)); info->offset = gpiod_hwgpio(desc); if (desc->name) @@ -2258,6 +2259,8 @@ static void gpio_desc_to_lineinfo(struct gpio_desc *desc, debounce_period_us; info->num_attrs++; } + + return 0; } struct gpio_chardev_data { @@ -2309,6 +2312,7 @@ static int lineinfo_get_v1(struct gpio_chardev_data *cdev, void __user *ip, struct gpio_desc *desc; struct gpioline_info lineinfo; struct gpio_v2_line_info lineinfo_v2; + int ret; if (copy_from_user(&lineinfo, ip, sizeof(lineinfo))) return -EFAULT; @@ -2326,7 +2330,10 @@ static int lineinfo_get_v1(struct gpio_chardev_data *cdev, void __user *ip, return -EBUSY; } - gpio_desc_to_lineinfo(desc, &lineinfo_v2, false); + ret = gpio_desc_to_lineinfo(desc, &lineinfo_v2, false); + if (ret) + return ret; + gpio_v2_line_info_to_v1(&lineinfo_v2, &lineinfo); if (copy_to_user(ip, &lineinfo, sizeof(lineinfo))) { @@ -2344,6 +2351,7 @@ static int lineinfo_get(struct gpio_chardev_data *cdev, void __user *ip, { struct gpio_desc *desc; struct gpio_v2_line_info lineinfo; + int ret; if (copy_from_user(&lineinfo, ip, sizeof(lineinfo))) return -EFAULT; @@ -2363,7 +2371,10 @@ static int lineinfo_get(struct gpio_chardev_data *cdev, void __user *ip, if (test_and_set_bit(lineinfo.offset, cdev->watched_lines)) return -EBUSY; } - gpio_desc_to_lineinfo(desc, &lineinfo, false); + + ret = gpio_desc_to_lineinfo(desc, &lineinfo, false); + if (ret) + return ret; if (copy_to_user(ip, &lineinfo, sizeof(lineinfo))) { if (watch) @@ -2489,6 +2500,7 @@ static int lineinfo_changed_notify(struct notifier_block *nb, struct lineinfo_changed_ctx *ctx; struct gpio_desc *desc = data; struct file *fp; + int ret; if (!test_bit(gpiod_hwgpio(desc), cdev->watched_lines)) return NOTIFY_DONE; @@ -2519,7 +2531,13 @@ static int lineinfo_changed_notify(struct notifier_block *nb, ctx->chg.event_type = action; ctx->chg.timestamp_ns = ktime_get_ns(); - gpio_desc_to_lineinfo(desc, &ctx->chg.info, true); + + ret = gpio_desc_to_lineinfo(desc, &ctx->chg.info, true); + if (ret) { + fput(fp); + return NOTIFY_DONE; + } + /* Keep the GPIO device alive until we emit the event. */ ctx->gdev = gpio_device_get(desc->gdev); ctx->cdev = cdev; diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 66d2325bfae8..772c6f04f489 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -1029,6 +1029,13 @@ int gpiochip_add_hog(struct gpio_chip *gc, struct fwnode_handle *fwnode) ret = of_gpiochip_get_lflags(gc, &gpiospec, &lflags); if (ret) return ret; + + /* + * If no line-name property is present, fall back to the OF + * node name as in the previous implementation. + */ + if (!name) + name = to_of_node(fwnode)->name; } else { /* * GPIO_ACTIVE_LOW is currently the only lookup flag diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c index e475bf2bd1a0..f7dc385e6582 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c @@ -318,7 +318,7 @@ static int acp_hw_init(struct amdgpu_ip_block *ip_block) r = device_for_each_child(adev->acp.parent, &adev->acp.acp_genpd->gpd, acp_genpd_add_device); if (r) - goto failure; + goto failure_remove_mfd; break; } default: @@ -443,7 +443,7 @@ static int acp_hw_init(struct amdgpu_ip_block *ip_block) r = device_for_each_child(adev->acp.parent, &adev->acp.acp_genpd->gpd, acp_genpd_add_device); if (r) - goto failure; + goto failure_remove_mfd; } /* Assert Soft reset of ACP */ @@ -461,7 +461,7 @@ static int acp_hw_init(struct amdgpu_ip_block *ip_block) if (--count == 0) { dev_err(&adev->pdev->dev, "Failed to reset ACP\n"); r = -ETIMEDOUT; - goto failure; + goto failure_remove_mfd; } udelay(100); } @@ -479,7 +479,7 @@ static int acp_hw_init(struct amdgpu_ip_block *ip_block) if (--count == 0) { dev_err(&adev->pdev->dev, "Failed to reset ACP\n"); r = -ETIMEDOUT; - goto failure; + goto failure_remove_mfd; } udelay(100); } @@ -489,6 +489,11 @@ static int acp_hw_init(struct amdgpu_ip_block *ip_block) cgs_write_register(adev->acp.cgs_device, mmACP_SOFT_RESET, val); return 0; +failure_remove_mfd: + device_for_each_child(adev->acp.parent, NULL, + acp_genpd_remove_device); + mfd_remove_devices(adev->acp.parent); + failure: kfree(adev->acp.i2s_pdata); kfree(adev->acp.acp_res); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c index ad21986d7b1b..16e0640570d9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c @@ -1374,6 +1374,8 @@ static int amdgpu_discovery_sysfs_ips(struct amdgpu_device *adev, ip_hw_instance->kobj.kset = &ip_hw_id->hw_id_kset; res = kobject_add(&ip_hw_instance->kobj, NULL, "%d", ip_hw_instance->num_instance); + if (res) + kobject_put(&ip_hw_instance->kobj); next_ip: if (reg_base_64) ip_offset += struct_size(ip, base_address_64, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index 9c5e93cd3ee6..4c4dce9c9942 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -2680,6 +2680,14 @@ static int amdgpu_pmops_suspend_noirq(struct device *dev) struct amdgpu_device *adev = drm_to_adev(drm_dev); int r; + /* + * A GPU parked by vga_switcheroo has no power and no PCIe link, so the + * ASIC reset below would fail and abort the whole noirq suspend phase. + * Bail out like amdgpu_device_prepare/suspend/resume() already do. + */ + if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF) + return 0; + if (amdgpu_acpi_should_gpu_reset(adev)) { amdgpu_device_lock_reset_domain(adev->reset_domain); r = amdgpu_asic_reset(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c index 3043ad041bb4..3017ddcefbe2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c @@ -724,8 +724,14 @@ void amdgpu_ring_set_fence_errors_and_reemit(struct amdgpu_ring *ring, bool is_guilty_fence; bool is_guilty_context; - last_seq = amdgpu_fence_read(ring) & ring->fence_drv.num_fences_mask; - seq = ring->fence_drv.sync_seq & ring->fence_drv.num_fences_mask; + last_seq = amdgpu_fence_read(ring); + seq = ring->fence_drv.sync_seq; + + /* Hardware completion can precede the interrupt which signals fences. */ + if (last_seq == seq) { + amdgpu_fence_process(ring); + return; + } /* If there is nothing to reemit, return early and set an error on the fence * if applicable. If all of the fences are siganlled, this will be a nop. @@ -733,7 +739,9 @@ void amdgpu_ring_set_fence_errors_and_reemit(struct amdgpu_ring *ring, * we are skipping it on purpose. */ if (!ring->ring_backup_entries_to_copy) { - amdgpu_fence_driver_force_completion(ring, &guilty_fence->base); + if (guilty_fence) + amdgpu_fence_driver_force_completion(ring, + &guilty_fence->base); return; } ring->reemit = true; @@ -741,9 +749,8 @@ void amdgpu_ring_set_fence_errors_and_reemit(struct amdgpu_ring *ring, spin_lock_irqsave(&ring->fence_drv.lock, flags); do { last_seq++; - last_seq &= ring->fence_drv.num_fences_mask; - ptr = &ring->fence_drv.fences[last_seq]; + ptr = &ring->fence_drv.fences[last_seq & ring->fence_drv.num_fences_mask]; rcu_read_lock(); unprocessed = rcu_dereference(*ptr); @@ -800,26 +807,24 @@ void amdgpu_ring_backup_unprocessed_commands(struct amdgpu_ring *ring, struct amdgpu_fence *fence; u32 seq, last_seq; - last_seq = amdgpu_fence_read(ring) & ring->fence_drv.num_fences_mask; - seq = ring->fence_drv.sync_seq & ring->fence_drv.num_fences_mask; + last_seq = amdgpu_fence_read(ring); + seq = ring->fence_drv.sync_seq; ring->ring_backup_entries_to_copy = 0; - /* if we've already seen this fence, return early. - * ring->ring_backup_entries_to_copy is set to 0 so - * the reemit helper will return early as well to - * avoid getting stuck in a reemit loop. + /* + * Avoid replay loops for a repeated guilty fence. NULL identifies a + * collateral ring, whose pending commands still need backup. */ - if (ring->guilty_fence == guilty_fence) { + if (guilty_fence && ring->guilty_fence == guilty_fence) { ring->guilty_fence = NULL; return; } ring->guilty_fence = guilty_fence; - do { + while (last_seq != seq) { last_seq++; - last_seq &= ring->fence_drv.num_fences_mask; - ptr = &ring->fence_drv.fences[last_seq]; + ptr = &ring->fence_drv.fences[last_seq & ring->fence_drv.num_fences_mask]; rcu_read_lock(); unprocessed = rcu_dereference(*ptr); @@ -829,7 +834,7 @@ void amdgpu_ring_backup_unprocessed_commands(struct amdgpu_ring *ring, amdgpu_ring_backup_unprocessed_command(ring, fence); } rcu_read_unlock(); - } while (last_seq != seq); + } } struct amdgpu_fence * diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index c509dc45ba19..a3777ceb638f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -2367,7 +2367,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev) void amdgpu_ttm_enable_buffer_funcs(struct amdgpu_device *adev) { struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); - u32 num_clear_entities, num_move_entities; + u32 num_clear_entities, num_move_entities, sdma_ip_version; int r, i, j; if (!adev->mman.initialized || amdgpu_in_reset(adev) || @@ -2392,6 +2392,12 @@ void amdgpu_ttm_enable_buffer_funcs(struct amdgpu_device *adev) num_clear_entities = MIN(adev->mman.num_buffer_funcs_scheds, TTM_NUM_MOVE_FENCES); num_move_entities = MIN(adev->mman.num_buffer_funcs_scheds, TTM_NUM_MOVE_FENCES); + /* TODO: workaround for DCC corruption when moving BOs from multiple queues at + * the same time: use a single queue until the root cause is identified and fixed. + */ + sdma_ip_version = amdgpu_ip_version(adev, SDMA0_HWIP, 0); + if (sdma_ip_version == IP_VERSION(7, 0, 0) || sdma_ip_version == IP_VERSION(7, 0, 1)) + num_move_entities = 1; adev->mman.clear_entities = kzalloc_objs(struct amdgpu_ttm_buffer_entity, num_clear_entities); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c index a7ed4923857b..e80c49a1da60 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c @@ -3812,6 +3812,7 @@ static int gfx_v11_0_cp_gfx_resume(struct amdgpu_device *adev) rb_bufsz = order_base_2(ring->ring_size / 8); tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz); tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); + tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, KMD_QUEUE, 1); WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp); /* Initialize the ring buffer's write pointers */ @@ -4220,6 +4221,7 @@ static int gfx_v11_0_gfx_mqd_init(struct amdgpu_device *adev, void *m, tmp = regCP_GFX_HQD_CNTL_DEFAULT; tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BUFSZ, rb_bufsz); tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BLKSZ, rb_bufsz - 2); + tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, KMD_QUEUE, prop->kernel_queue); #ifdef __BIG_ENDIAN tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, BUF_SWAP, 1); #endif diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c index ec1e142549c5..5069148f53f8 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c @@ -2771,6 +2771,7 @@ static int gfx_v12_0_cp_gfx_resume(struct amdgpu_device *adev) rb_bufsz = order_base_2(ring->ring_size / 8); tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz); tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); + tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, KMD_QUEUE, 1); WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp); /* Initialize the ring buffer's write pointers */ @@ -3063,6 +3064,7 @@ static int gfx_v12_0_gfx_mqd_init(struct amdgpu_device *adev, void *m, tmp = regCP_GFX_HQD_CNTL_DEFAULT; tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BUFSZ, rb_bufsz); tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BLKSZ, rb_bufsz - 2); + tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, KMD_QUEUE, prop->kernel_queue); #ifdef __BIG_ENDIAN tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, BUF_SWAP, 1); #endif diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c index 95e939363db6..a5553d181d97 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c @@ -322,6 +322,14 @@ static const u32 verde_rlc_save_restore_register_list[] = 0x00000000 }; +static void gfx_v6_0_free_microcode(struct amdgpu_device *adev) +{ + amdgpu_ucode_release(&adev->gfx.pfp_fw); + amdgpu_ucode_release(&adev->gfx.me_fw); + amdgpu_ucode_release(&adev->gfx.ce_fw); + amdgpu_ucode_release(&adev->gfx.rlc_fw); +} + static int gfx_v6_0_init_microcode(struct amdgpu_device *adev) { const char *chip_name; @@ -3254,6 +3262,7 @@ static int gfx_v6_0_sw_fini(struct amdgpu_ip_block *ip_block) amdgpu_ring_fini(&adev->gfx.compute_ring[i]); amdgpu_gfx_rlc_fini(adev); + gfx_v6_0_free_microcode(adev); return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c index d7f56bbcfd44..8e3fe16668c9 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c @@ -331,7 +331,7 @@ static void gmc_v12_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, return; } - gmc_v12_0_flush_vm_hub(adev, vmid, vmhub, 0); + gmc_v12_0_flush_vm_hub(adev, vmid, vmhub, flush_type); return; } diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c index 3349e8bc1c20..065012b5c6c9 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c @@ -420,7 +420,7 @@ static void gmc_v12_1_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, return; } - gmc_v12_1_flush_vm_hub(adev, vmid, vmhub, 0); + gmc_v12_1_flush_vm_hub(adev, vmid, vmhub, flush_type); return; } diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c index 321c67e01443..89b55d437ee8 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c @@ -1347,10 +1347,6 @@ static int sdma_v7_1_sw_init(struct amdgpu_ip_block *ip_block) else DRM_ERROR("Failed to allocated memory for SDMA IP Dump\n"); -#ifdef CONFIG_DRM_AMDGPU_NAVI3X_USERQ - adev->userq_funcs[AMDGPU_HW_IP_DMA] = &userq_mes_funcs; -#endif - return r; } diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c index 2cd6d7d77b1c..a0435fd5525b 100644 --- a/drivers/gpu/drm/amd/amdgpu/vi.c +++ b/drivers/gpu/drm/amd/amdgpu/vi.c @@ -1397,6 +1397,7 @@ struct vi_reset_quirk { static const struct vi_reset_quirk vi_reset_quirks[] = { { 0x67ef, PCI_VENDOR_ID_APPLE, 0x0190, 0xe3 }, /* Radeon Pro 555X */ { 0x67ef, PCI_VENDOR_ID_APPLE, 0x018f, 0xc2 }, /* Radeon Pro 560X */ + { 0x67ef, PCI_VENDOR_ID_APPLE, 0x017a, 0xc7 }, /* Radeon Pro 555 */ }; static bool vi_need_reset_on_init(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index c521ec07bf3c..a99dbedf3cd4 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -1757,12 +1757,15 @@ static int svm_range_validate_and_map(struct mm_struct *mm, } /* - * If prange with always mapped flag, update mapping on GPUs with - * ACCESS attribute + * If prange with always mapped flag is not mapped or migrated + * yet, update mapping on GPUs with ACCESS attribute. Don't add + * them otherwise, ACCESS GPUs may be on different XGMI hives, + * that would force hmm_range_fault to migrate VRAM back to ram. */ - if (prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED) - bitmap_or(ctx->bitmap, ctx->bitmap, prange->bitmap_access, - MAX_GPU_INSTANCE); + if (bitmap_empty(ctx->bitmap, MAX_GPU_INSTANCE) && + prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED) + bitmap_copy(ctx->bitmap, prange->bitmap_access, + MAX_GPU_INSTANCE); } else { bitmap_or(ctx->bitmap, prange->bitmap_access, prange->bitmap_aip, MAX_GPU_INSTANCE); diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index cd5cbb851075..a5e19d95b25a 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -454,6 +454,8 @@ STATIC_IFN_KUNIT int amdgpu_dm_init_power_module(struct amdgpu_display_manager * init_data[i].use_custom_backlight_caps = false; init_data[i].custom_backlight_caps_config_no = 0; init_data[i].use_linear_backlight_curve = false; + if (amdgpu_dc_debug_mask & DC_DISABLE_CUSTOM_BRIGHTNESS_CURVE) + init_data[i].use_linear_backlight_curve = true; init_data[i].def_varibright_enable = 0; init_data[i].varibright_level = 0; /* @@ -1626,9 +1628,14 @@ static int dm_suspend(struct amdgpu_ip_block *ip_block) dc_allow_idle_optimizations(adev->dm.dc, false); dm->cached_dc_state = dc_state_create_copy(dm->dc->current_state); + if (!dm->cached_dc_state) { + drm_err(adev_to_drm(adev), + "Failed to allocate cached DC state during suspend\n"); + mutex_unlock(&dm->dc_lock); + return -ENOMEM; + } - if (dm->cached_dc_state) - dm_gpureset_toggle_interrupts(adev, dm->cached_dc_state, false); + dm_gpureset_toggle_interrupts(adev, dm->cached_dc_state, false); res = amdgpu_dm_commit_zero_streams(dm->dc); if (res != DC_OK) { diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_backlight.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_backlight.c index 8dc57903de9e..cb74b00723fd 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_backlight.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_backlight.c @@ -39,6 +39,7 @@ #include "modules/inc/mod_power.h" #include +#include #include #include #include @@ -127,6 +128,68 @@ static inline u32 scale_fw_to_input(int min, int max, u64 input) return min + DIV_ROUND_CLOSEST_ULL(input * (max - min), AMDGPU_MAX_BL_LEVEL); } +static u32 backlight_signal_to_luminance(const struct amdgpu_dm_backlight_caps *caps, + u32 signal) +{ + u32 lower_signal, upper_signal; + u32 lower_lum, upper_lum; + int i; + + if (!caps->data_points) + return DIV_ROUND_CLOSEST(signal * 100, AMDGPU_MAX_BL_LEVEL); + + if (signal < caps->luminance_data[0].input_signal) { + lower_signal = 0; + lower_lum = 0; + upper_signal = caps->luminance_data[0].input_signal; + upper_lum = caps->luminance_data[0].luminance; + } else if (signal >= + caps->luminance_data[caps->data_points - 1].input_signal) { + lower_signal = caps->luminance_data[caps->data_points - 1].input_signal; + lower_lum = caps->luminance_data[caps->data_points - 1].luminance; + upper_signal = caps->max_input_signal; + upper_lum = 100; + } else { + for (i = 1; + i < caps->data_points && + signal > caps->luminance_data[i].input_signal; + i++) + ; + + lower_signal = caps->luminance_data[i - 1].input_signal; + upper_signal = caps->luminance_data[i].input_signal; + lower_lum = caps->luminance_data[i - 1].luminance; + upper_lum = caps->luminance_data[i].luminance; + } + + if (upper_signal == lower_signal) + return upper_lum; + + return lower_lum + DIV_ROUND_CLOSEST((upper_lum - lower_lum) * + (signal - lower_signal), + upper_signal - lower_signal); +} + +static u32 convert_brightness_to_millipercent(const struct amdgpu_dm_backlight_caps *caps, + u32 user_brightness) +{ + unsigned int min, max; + u32 signal, luminance; + + if (!get_brightness_range(caps, &min, &max)) + return DIV_ROUND_CLOSEST_ULL((u64)user_brightness * 100000, + MAX_BACKLIGHT_LEVEL); + + user_brightness = clamp(user_brightness, min, max); + if (amdgpu_dc_debug_mask & DC_DISABLE_CUSTOM_BRIGHTNESS_CURVE) + return DIV_ROUND_CLOSEST_ULL((u64)user_brightness * 100000, max); + + signal = scale_input_to_fw(max, user_brightness); + luminance = backlight_signal_to_luminance(caps, signal); + + return min_t(u32, luminance * 1000, 100000); +} + STATIC_IFN_KUNIT void convert_custom_brightness(const struct amdgpu_dm_backlight_caps *caps, unsigned int min, unsigned int max, @@ -269,10 +332,10 @@ void amdgpu_dm_backlight_set_level(struct amdgpu_display_manager *dm, struct amdgpu_dm_backlight_caps *caps; struct dc_link *link; u32 brightness = 0; + u32 millipercent = 0; bool rc = false, reallow_idle = false; struct drm_connector *connector; struct dc_stream_state *stream; - unsigned int min, max; list_for_each_entry(connector, &dm->ddev->mode_config.connector_list, head) { struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector); @@ -295,12 +358,21 @@ void amdgpu_dm_backlight_set_level(struct amdgpu_display_manager *dm, /* update scratch register */ if (bl_idx == 0) amdgpu_atombios_scratch_regs_set_backlight_level(dm->adev, dm->brightness[bl_idx]); - brightness = convert_brightness_from_user(caps, dm->brightness[bl_idx]); - link = (struct dc_link *)dm->backlight_link[bl_idx]; - /* Apply brightness quirk */ - if (caps->brightness_mask) - brightness |= caps->brightness_mask; + link = (struct dc_link *)dm->backlight_link[bl_idx]; + stream = dm_find_stream_with_link(dm, link); + if (!stream) + return; + + if (caps->aux_support) { + brightness = convert_brightness_from_user(caps, dm->brightness[bl_idx]); + /* Apply brightness quirk */ + if (caps->brightness_mask) + brightness |= caps->brightness_mask; + } else { + millipercent = convert_brightness_to_millipercent(caps, + user_brightness); + } if (trace_amdgpu_dm_brightness_enabled()) { trace_amdgpu_dm_brightness(__builtin_return_address(0), @@ -310,10 +382,6 @@ void amdgpu_dm_backlight_set_level(struct amdgpu_display_manager *dm, power_supply_is_system_supplied() > 0); } - stream = dm_find_stream_with_link(dm, link); - if (!stream) - return; - mutex_lock(&dm->dc_lock); if (dm->dc->caps.ips_support && dm->dc->ctx->dmub_srv->idle_allowed) { dc_allow_idle_optimizations(dm->dc, false); @@ -325,10 +393,8 @@ void amdgpu_dm_backlight_set_level(struct amdgpu_display_manager *dm, AUX_BL_DEFAULT_TRANSITION_TIME_MS, false, true); } else { /* power module uses millipercent */ - get_brightness_range(caps, &min, &max); - brightness = DIV_ROUND_CLOSEST(brightness * 100, (max - min)) * 1000; rc = mod_power_set_backlight_percent(dm->power_module, stream, - brightness, 0, false); + millipercent, 0, false); } /* diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.c index 25f168a8ea31..1861a197129d 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_connector.c @@ -732,6 +732,24 @@ get_aspect_ratio(const struct drm_display_mode *mode_in) } EXPORT_IF_KUNIT(get_aspect_ratio); +/* + * CTA-861 5.1: RGB video formats default to limited range except 640x480 (VIC 1). + * With Broadcast RGB left at Automatic follow that default, as i915 does. + */ +static bool rgb_output_is_limited_range(const struct dc_crtc_timing *dc_crtc_timing, + const struct drm_connector_state *connector_state) +{ + switch (connector_state->hdmi.broadcast_rgb) { + case DRM_HDMI_BROADCAST_RGB_FULL: + return false; + case DRM_HDMI_BROADCAST_RGB_LIMITED: + return true; + default: + return connector_state->connector->display_info.is_hdmi && + dc_crtc_timing->vic > 1; + } +} + enum dc_color_space amdgpu_dm_get_output_color_space(const struct dc_crtc_timing *dc_crtc_timing, const struct drm_connector_state *connector_state) @@ -757,7 +775,7 @@ amdgpu_dm_get_output_color_space(const struct dc_crtc_timing *dc_crtc_timing, case DRM_MODE_COLORIMETRY_BT2020_RGB: case DRM_MODE_COLORIMETRY_BT2020_YCC: if (dc_crtc_timing->pixel_encoding == PIXEL_ENCODING_RGB) { - if (connector_state->hdmi.broadcast_rgb == DRM_HDMI_BROADCAST_RGB_LIMITED) + if (rgb_output_is_limited_range(dc_crtc_timing, connector_state)) color_space = COLOR_SPACE_2020_RGB_LIMITEDRANGE; else color_space = COLOR_SPACE_2020_RGB_FULLRANGE; @@ -769,7 +787,7 @@ amdgpu_dm_get_output_color_space(const struct dc_crtc_timing *dc_crtc_timing, default: if (dc_crtc_timing->pixel_encoding == PIXEL_ENCODING_RGB) { color_space = COLOR_SPACE_SRGB; - if (connector_state->hdmi.broadcast_rgb == DRM_HDMI_BROADCAST_RGB_LIMITED) + if (rgb_output_is_limited_range(dc_crtc_timing, connector_state)) color_space = COLOR_SPACE_SRGB_LIMITED; /* * 27030khz is the separation point between HDTV and SDTV @@ -2262,8 +2280,8 @@ amdgpu_dm_create_validate_stream_for_sink(struct drm_connector *connector, * below gate which of these entries are actually attempted. */ static const enum dc_pixel_encoding encoding_order[] = { - PIXEL_ENCODING_YCBCR444, PIXEL_ENCODING_RGB, + PIXEL_ENCODING_YCBCR444, PIXEL_ENCODING_YCBCR422, PIXEL_ENCODING_YCBCR420, }; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c index a5b15bef8aa0..ee3f3df6bb8d 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c @@ -1238,7 +1238,8 @@ STATIC_IFN_KUNIT bool get_conv_frl_bw(struct amdgpu_dm_connector *aconnector, unsigned int max_sink_bw_in_kbps = 0; unsigned int dsc_max_sink_bw_in_kbps = 0; - if (aconnector->dc_link->dc->caps.dp_hdmi21_pcon_support && + if (aconnector->dc_sink && + aconnector->dc_link->dc->caps.dp_hdmi21_pcon_support && aconnector->mst_downstream_port_caps.bytes.byte0.bits.DWN_STRM_PORTX_TYPE == DOWN_STREAM_DETAILED_HDMI) { max_conv_bw_in_kbps = dc_link_bw_kbps_from_raw_frl_link_rate_data( aconnector->dc_link->dc, @@ -2019,6 +2020,9 @@ enum dc_status dm_dp_mst_is_port_support_mode( struct dc_dsc_config_options dsc_options = {0}; uint32_t stream_kbps; + if (!aconnector->dc_sink) + return DC_FAIL_BANDWIDTH_VALIDATE; + /* DSC unnecessary case * Check if timing could be supported within end-to-end BW */ diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_connector_test.c b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_connector_test.c index e049d1a32979..df38f90687c9 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_connector_test.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/tests/amdgpu_dm_connector_test.c @@ -415,7 +415,9 @@ static void dm_test_output_color_space_default_rgb_full(struct kunit *test) { struct dc_crtc_timing timing = {}; struct drm_connector_state state = {}; + struct drm_connector *connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL); + state.connector = connector; timing.pixel_encoding = PIXEL_ENCODING_RGB; state.colorspace = DRM_MODE_COLORIMETRY_DEFAULT; state.hdmi.broadcast_rgb = DRM_HDMI_BROADCAST_RGB_AUTO; @@ -565,7 +567,9 @@ static void dm_test_output_color_space_bt2020_rgb(struct kunit *test) { struct dc_crtc_timing timing = {}; struct drm_connector_state state = {}; + struct drm_connector *connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL); + state.connector = connector; timing.pixel_encoding = PIXEL_ENCODING_RGB; state.colorspace = DRM_MODE_COLORIMETRY_BT2020_RGB; @@ -590,6 +594,110 @@ static void dm_test_output_color_space_bt2020_rgb_limited(struct kunit *test) (int)COLOR_SPACE_2020_RGB_LIMITEDRANGE); } +/** + * dm_test_output_color_space_default_rgb_auto_hdmi_cta - Automatic follows the CTA-861 default + * @test: The KUnit test context + */ +static void dm_test_output_color_space_default_rgb_auto_hdmi_cta(struct kunit *test) +{ + struct dc_crtc_timing timing = {}; + struct drm_connector_state state = {}; + struct drm_connector *connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL); + + connector->display_info.is_hdmi = true; + timing.pixel_encoding = PIXEL_ENCODING_RGB; + timing.vic = 32; + state.connector = connector; + state.colorspace = DRM_MODE_COLORIMETRY_DEFAULT; + state.hdmi.broadcast_rgb = DRM_HDMI_BROADCAST_RGB_AUTO; + + KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_get_output_color_space(&timing, &state), + (int)COLOR_SPACE_SRGB_LIMITED); +} + +/** + * dm_test_output_color_space_default_rgb_auto_hdmi_vga - 640x480 (VIC 1) stays full range + * @test: The KUnit test context + */ +static void dm_test_output_color_space_default_rgb_auto_hdmi_vga(struct kunit *test) +{ + struct dc_crtc_timing timing = {}; + struct drm_connector_state state = {}; + struct drm_connector *connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL); + + connector->display_info.is_hdmi = true; + timing.pixel_encoding = PIXEL_ENCODING_RGB; + timing.vic = 1; + state.connector = connector; + state.colorspace = DRM_MODE_COLORIMETRY_DEFAULT; + state.hdmi.broadcast_rgb = DRM_HDMI_BROADCAST_RGB_AUTO; + + KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_get_output_color_space(&timing, &state), + (int)COLOR_SPACE_SRGB); +} + +/** + * dm_test_output_color_space_default_rgb_auto_dp_cta - Automatic on a non-HDMI sink is full range + * @test: The KUnit test context + */ +static void dm_test_output_color_space_default_rgb_auto_dp_cta(struct kunit *test) +{ + struct dc_crtc_timing timing = {}; + struct drm_connector_state state = {}; + struct drm_connector *connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL); + + timing.pixel_encoding = PIXEL_ENCODING_RGB; + timing.vic = 32; + state.connector = connector; + state.colorspace = DRM_MODE_COLORIMETRY_DEFAULT; + state.hdmi.broadcast_rgb = DRM_HDMI_BROADCAST_RGB_AUTO; + + KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_get_output_color_space(&timing, &state), + (int)COLOR_SPACE_SRGB); +} + +/** + * dm_test_output_color_space_default_rgb_full_hdmi_cta - Full overrides the CTA-861 default + * @test: The KUnit test context + */ +static void dm_test_output_color_space_default_rgb_full_hdmi_cta(struct kunit *test) +{ + struct dc_crtc_timing timing = {}; + struct drm_connector_state state = {}; + struct drm_connector *connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL); + + connector->display_info.is_hdmi = true; + timing.pixel_encoding = PIXEL_ENCODING_RGB; + timing.vic = 32; + state.connector = connector; + state.colorspace = DRM_MODE_COLORIMETRY_DEFAULT; + state.hdmi.broadcast_rgb = DRM_HDMI_BROADCAST_RGB_FULL; + + KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_get_output_color_space(&timing, &state), + (int)COLOR_SPACE_SRGB); +} + +/** + * dm_test_output_color_space_bt2020_rgb_auto_hdmi_cta - BT.2020 RGB follows the same default + * @test: The KUnit test context + */ +static void dm_test_output_color_space_bt2020_rgb_auto_hdmi_cta(struct kunit *test) +{ + struct dc_crtc_timing timing = {}; + struct drm_connector_state state = {}; + struct drm_connector *connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL); + + connector->display_info.is_hdmi = true; + timing.pixel_encoding = PIXEL_ENCODING_RGB; + timing.vic = 32; + state.connector = connector; + state.colorspace = DRM_MODE_COLORIMETRY_BT2020_RGB; + state.hdmi.broadcast_rgb = DRM_HDMI_BROADCAST_RGB_AUTO; + + KUNIT_EXPECT_EQ(test, (int)amdgpu_dm_get_output_color_space(&timing, &state), + (int)COLOR_SPACE_2020_RGB_LIMITEDRANGE); +} + /** * dm_test_output_color_space_bt2020_ycc - Test Output color space bt2020 ycc * @test: The KUnit test context @@ -653,7 +761,9 @@ static void dm_test_output_color_space_bt2020_ycc_rgb_encoding(struct kunit *tes { struct dc_crtc_timing timing = {}; struct drm_connector_state state = {}; + struct drm_connector *connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL); + state.connector = connector; timing.pixel_encoding = PIXEL_ENCODING_RGB; state.colorspace = DRM_MODE_COLORIMETRY_BT2020_YCC; @@ -3597,6 +3707,7 @@ static struct dm_test_fill_ctx *dm_test_fill_ctx_alloc(struct kunit *test) ctx->conn_state = drmm_kzalloc(ctx->drm, sizeof(*ctx->conn_state), GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, ctx->conn_state); + ctx->conn_state->connector = &ctx->aconnector->base; ctx->stream = kunit_kzalloc(test, sizeof(*ctx->stream), GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, ctx->stream); ctx->mode = kunit_kzalloc(test, sizeof(*ctx->mode), GFP_KERNEL); @@ -4066,6 +4177,7 @@ static struct dm_test_stream_ctx *dm_test_stream_ctx_alloc(struct kunit *test) ctx->dm_state = kunit_kzalloc(test, sizeof(*ctx->dm_state), GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, ctx->dm_state); + ctx->dm_state->base.connector = &ctx->aconnector->base; ctx->dm_state->scaling = RMX_OFF; ctx->mode = kunit_kzalloc(test, sizeof(*ctx->mode), GFP_KERNEL); @@ -9818,6 +9930,11 @@ static struct kunit_case amdgpu_dm_connector_tests[] = { KUNIT_CASE(dm_test_output_color_space_oprgb), KUNIT_CASE(dm_test_output_color_space_bt2020_rgb), KUNIT_CASE(dm_test_output_color_space_bt2020_rgb_limited), + KUNIT_CASE(dm_test_output_color_space_default_rgb_auto_hdmi_cta), + KUNIT_CASE(dm_test_output_color_space_default_rgb_auto_hdmi_vga), + KUNIT_CASE(dm_test_output_color_space_default_rgb_auto_dp_cta), + KUNIT_CASE(dm_test_output_color_space_default_rgb_full_hdmi_cta), + KUNIT_CASE(dm_test_output_color_space_bt2020_rgb_auto_hdmi_cta), KUNIT_CASE(dm_test_output_color_space_bt2020_ycc), KUNIT_CASE(dm_test_output_color_space_default_ycbcr709_y_only), KUNIT_CASE(dm_test_output_color_space_default_ycbcr601), diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index c8e6e949ce79..cd31d8b1a527 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -1909,6 +1909,7 @@ struct dc_scratch_space { struct ddc_service *ddc; enum dp_panel_mode panel_mode; + bool panel_mode_initialized; bool aux_mode; /* Private to DC core */ diff --git a/drivers/gpu/drm/amd/display/dc/dml/Makefile b/drivers/gpu/drm/amd/display/dc/dml/Makefile index 91465ac05c97..eb4ce5b1ecfb 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dml/Makefile @@ -29,7 +29,7 @@ dml_ccflags := $(CC_FLAGS_FPU) dml_rcflags := $(CC_FLAGS_NO_FPU) ifneq ($(CONFIG_FRAME_WARN),0) - ifeq ($(filter y,$(CONFIG_KASAN)$(CONFIG_KCSAN)$(CONFIG_UBSAN)),y) + ifneq ($(filter y,$(CONFIG_KASAN) $(CONFIG_KCSAN) $(CONFIG_UBSAN)),) ifeq ($(CONFIG_CC_IS_CLANG)$(CONFIG_COMPILE_TEST),yy) frame_warn_limit := 4096 else diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/Makefile b/drivers/gpu/drm/amd/display/dc/dml2_0/Makefile index e8bd061be05c..2181079c89d0 100644 --- a/drivers/gpu/drm/amd/display/dc/dml2_0/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dml2_0/Makefile @@ -28,7 +28,7 @@ dml2_ccflags := $(CC_FLAGS_FPU) dml2_rcflags := $(CC_FLAGS_NO_FPU) ifneq ($(CONFIG_FRAME_WARN),0) - ifeq ($(filter y,$(CONFIG_KASAN)$(CONFIG_KCSAN)$(CONFIG_UBSAN)),y) + ifneq ($(filter y,$(CONFIG_KASAN) $(CONFIG_KCSAN) $(CONFIG_UBSAN)),) ifeq ($(CONFIG_CC_IS_CLANG)$(CONFIG_COMPILE_TEST),yy) frame_warn_limit := 4096 else diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c index e904c7dd320e..0e9b2b675eac 100644 --- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c +++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c @@ -93,6 +93,7 @@ void dp_set_panel_mode(struct dc_link *link, enum dp_panel_mode panel_mode) } link->panel_mode = panel_mode; + link->panel_mode_initialized = true; DC_LOG_DETECTION_DP_CAPS("%d eDP panel mode supported: %d, enabled: %d\n", link->link_index, link->dpcd_caps.panel_mode_edp, @@ -1322,7 +1323,8 @@ void edp_set_panel_assr(struct dc_link *link, struct pipe_ctx *pipe_ctx, result = cp_psp->funcs.enable_assr(cp_psp->handle, link); - if (!result && link->panel_mode != DP_PANEL_MODE_EDP) + if (!result && link->panel_mode_initialized && + link->panel_mode != DP_PANEL_MODE_EDP) *panel_mode = DP_PANEL_MODE_DEFAULT; } } diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c index d2d56a1c4b8b..65c7dae3ff66 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c @@ -448,6 +448,12 @@ static const struct dce_abm_mask abm_mask = { ABM_MASK_SH_LIST_DCE110(_MASK) }; +static const struct dc_debug_options debug_defaults = { 0 }; + +static const struct dc_check_config config_defaults = { + .enable_legacy_fast_update = true, +}; + #define CTX ctx #define REG(reg) mm ## reg @@ -937,6 +943,8 @@ static bool dce60_construct( dc->caps.max_cursor_size = 64; dc->caps.dual_link_dvi = true; dc->caps.extended_aux_timeout_support = false; + dc->debug = debug_defaults; + dc->check_config = config_defaults; /************************************************* * Create resources * @@ -1134,6 +1142,8 @@ static bool dce61_construct( dc->caps.i2c_speed_in_khz = 40; dc->caps.max_cursor_size = 64; dc->caps.is_apu = true; + dc->debug = debug_defaults; + dc->check_config = config_defaults; /************************************************* * Create resources * @@ -1331,7 +1341,8 @@ static bool dce64_construct( dc->caps.max_downscale_ratio = 200; dc->caps.i2c_speed_in_khz = 40; dc->caps.max_cursor_size = 64; - dc->caps.is_apu = true; + dc->debug = debug_defaults; + dc->check_config = config_defaults; /************************************************* * Create resources * diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c index 26820d35478a..2f4de697ec4b 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c @@ -1148,6 +1148,8 @@ static bool dce81_construct( dc->caps.max_cursor_size = 128; dc->caps.min_horizontal_blanking_period = 80; dc->caps.is_apu = true; + dc->debug = debug_defaults; + dc->check_config = config_defaults; /************************************************* * Create resources * diff --git a/drivers/gpu/drm/amd/display/modules/power/power.c b/drivers/gpu/drm/amd/display/modules/power/power.c index 2f9690e65ca9..d900be8cb5fc 100644 --- a/drivers/gpu/drm/amd/display/modules/power/power.c +++ b/drivers/gpu/drm/amd/display/modules/power/power.c @@ -328,6 +328,7 @@ bool mod_power_add_stream(struct mod_power *mod_power, core_power->map[core_power->num_entities].psr_enabled = 0; core_power->map[core_power->num_entities].psr_events = psr_event_vsync; core_power->map[core_power->num_entities].psr_power_opt = 0; + core_power->map[core_power->num_entities].replay_events = replay_event_vsync; core_power->num_entities++; return true; } @@ -387,6 +388,7 @@ bool mod_power_remove_stream(struct mod_power *mod_power, core_power->map[i].psr_enabled = core_power->map[i + 1].psr_enabled; core_power->map[i].psr_events = core_power->map[i + 1].psr_events; core_power->map[i].psr_power_opt = core_power->map[i + 1].psr_power_opt; + core_power->map[i].replay_events = core_power->map[i + 1].replay_events; memcpy(core_power->map[i].psr_context, core_power->map[i + 1].psr_context, sizeof(struct mod_power_psr_context)); memset(core_power->map[i + 1].psr_context, 0, sizeof(struct mod_power_psr_context)); diff --git a/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c b/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c index 6cc414a8a4de..f186aae9984d 100644 --- a/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c +++ b/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c @@ -5263,7 +5263,8 @@ static int si_init_smc_table(struct amdgpu_device *adev) break; } - if (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC) + if ((adev->flags & AMD_IS_MOBILITY) && + (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC)) table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC; if (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT) { @@ -7237,6 +7238,19 @@ static void si_parse_pplib_non_clock_info(struct amdgpu_device *adev, adev->pm.dpm.uvd_ps = rps; } +static void si_update_limits(struct amdgpu_clock_and_voltage_limits *limits, + struct rv7xx_pl *pl) +{ + if (pl->sclk > limits->sclk) + limits->sclk = pl->sclk; + if (pl->mclk > limits->mclk) + limits->mclk = pl->mclk; + if (pl->vddc > limits->vddc) + limits->vddc = pl->vddc; + if (pl->vddci > limits->vddci) + limits->vddci = pl->vddci; +} + static void si_parse_pplib_clock_info(struct amdgpu_device *adev, struct amdgpu_ps *rps, int index, union pplib_clock_info *clock_info) @@ -7245,7 +7259,6 @@ static void si_parse_pplib_clock_info(struct amdgpu_device *adev, struct evergreen_power_info *eg_pi = evergreen_get_pi(adev); struct si_power_info *si_pi = si_get_pi(adev); struct si_ps *ps = si_get_ps(rps); - struct amdgpu_clock_and_voltage_limits *limits; u16 leakage_voltage; struct rv7xx_pl *pl = &ps->performance_levels[index]; int ret; @@ -7310,26 +7323,14 @@ static void si_parse_pplib_clock_info(struct amdgpu_device *adev, * VBIOS can contain conflicting values between: * - the maximum allowed clocks and voltages on AC or DC * - the clocks and voltages in power states on AC or DC + * + * Assume that the AC limits are the maximum of all power states, + * and the DC limits are the maximum of battery power states. */ + si_update_limits(&adev->pm.dpm.dyn_state.max_clock_voltage_on_ac, pl); if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == - ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) - limits = &adev->pm.dpm.dyn_state.max_clock_voltage_on_ac; - else if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == ATOM_PPLIB_CLASSIFICATION_UI_BATTERY) - limits = &adev->pm.dpm.dyn_state.max_clock_voltage_on_dc; - else - limits = NULL; - - if (limits) { - if (pl->sclk > limits->sclk) - limits->sclk = pl->sclk; - if (pl->mclk > limits->mclk) - limits->mclk = pl->mclk; - if (pl->vddc > limits->vddc) - limits->vddc = pl->vddc; - if (pl->vddci > limits->vddci) - limits->vddci = pl->vddci; - } + si_update_limits(&adev->pm.dpm.dyn_state.max_clock_voltage_on_dc, pl); } union pplib_power_state { diff --git a/drivers/gpu/drm/bridge/samsung-dsim.c b/drivers/gpu/drm/bridge/samsung-dsim.c index ea2837384774..480e568dd2c9 100644 --- a/drivers/gpu/drm/bridge/samsung-dsim.c +++ b/drivers/gpu/drm/bridge/samsung-dsim.c @@ -1861,7 +1861,7 @@ static int samsung_dsim_register_te_irq(struct samsung_dsim *dsi, struct device int te_gpio_irq; int ret; - dsi->te_gpio = devm_gpiod_get_optional(dev, "te", GPIOD_IN); + dsi->te_gpio = gpiod_get_optional(dev, "te", GPIOD_IN); if (!dsi->te_gpio) return 0; else if (IS_ERR(dsi->te_gpio)) diff --git a/drivers/gpu/drm/clients/drm_fbdev_client.c b/drivers/gpu/drm/clients/drm_fbdev_client.c index 4300d51fce32..eec463d19943 100644 --- a/drivers/gpu/drm/clients/drm_fbdev_client.c +++ b/drivers/gpu/drm/clients/drm_fbdev_client.c @@ -42,6 +42,14 @@ static int drm_fbdev_client_restore(struct drm_client_dev *client, bool force) { struct drm_fb_helper *fb_helper = drm_fb_helper_from_client(client); + /* + * The client is registered before the initial fbdev probe. + * If probing failed, the client remains registered but there + * is no valid fbdev framebuffer to restore. + */ + if (!fb_helper->info || !fb_helper->fb) + return 0; + drm_fb_helper_restore_fbdev_mode_unlocked(fb_helper, force); return 0; diff --git a/drivers/gpu/drm/i915/display/intel_cursor.c b/drivers/gpu/drm/i915/display/intel_cursor.c index 90173040d825..4afd275de86e 100644 --- a/drivers/gpu/drm/i915/display/intel_cursor.c +++ b/drivers/gpu/drm/i915/display/intel_cursor.c @@ -537,7 +537,8 @@ static void i9xx_cursor_disable_sel_fetch_arm(struct intel_dsb *dsb, struct intel_display *display = to_intel_display(plane); enum pipe pipe = plane->pipe; - if (!crtc_state->enable_psr2_sel_fetch) + if (!crtc_state->enable_psr2_sel_fetch && + !crtc_state->clear_psr2_sel_fetch) return; intel_de_write_dsb(display, dsb, SEL_FETCH_CUR_CTL(pipe), 0); @@ -570,8 +571,10 @@ static void i9xx_cursor_update_sel_fetch_arm(struct intel_dsb *dsb, struct intel_display *display = to_intel_display(plane); enum pipe pipe = plane->pipe; - if (!crtc_state->enable_psr2_sel_fetch) + if (!crtc_state->enable_psr2_sel_fetch) { + i9xx_cursor_disable_sel_fetch_arm(dsb, plane, crtc_state); return; + } if (drm_rect_height(&plane_state->psr2_sel_fetch_area) > 0) { if (crtc_state->enable_psr2_su_region_et) { diff --git a/drivers/gpu/drm/i915/display/intel_display_params.c b/drivers/gpu/drm/i915/display/intel_display_params.c index 2aed110c5b09..ca0ef466bb10 100644 --- a/drivers/gpu/drm/i915/display/intel_display_params.c +++ b/drivers/gpu/drm/i915/display/intel_display_params.c @@ -120,6 +120,10 @@ intel_display_param_named_unsafe(enable_psr, int, 0400, "(0=disabled, 1=enable up to PSR1, 2=enable up to PSR2) " "Default: -1 (use per-chip default)"); +intel_display_param_named_unsafe(enable_dc_balance, bool, 0400, + "Enable VRR DC balance (0=disabled, 1=enabled). " + "Default: 0 (disabled)"); + intel_display_param_named_unsafe(enable_panel_replay, int, 0400, "Enable Panel Replay (0=disabled, 1=enabled). Default: -1 (use per-chip default)"); diff --git a/drivers/gpu/drm/i915/display/intel_display_params.h b/drivers/gpu/drm/i915/display/intel_display_params.h index b95ecf728daa..16759e94c025 100644 --- a/drivers/gpu/drm/i915/display/intel_display_params.h +++ b/drivers/gpu/drm/i915/display/intel_display_params.h @@ -46,6 +46,7 @@ struct drm_printer; param(bool, enable_dp_mst, true, 0600) \ param(int, enable_fbc, -1, 0600) \ param(int, enable_psr, -1, 0600) \ + param(bool, enable_dc_balance, false, 0600) \ param(int, enable_panel_replay, -1, 0600) \ param(bool, psr_safest_params, false, 0400) \ param(bool, enable_psr2_sel_fetch, true, 0400) \ diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h index 5f0fe18c0614..5cfc123c86d1 100644 --- a/drivers/gpu/drm/i915/display/intel_display_types.h +++ b/drivers/gpu/drm/i915/display/intel_display_types.h @@ -1187,6 +1187,8 @@ struct intel_crtc_state { bool has_sel_update; bool enable_psr2_sel_fetch; bool enable_psr2_su_region_et; + /* Drop the stale selective fetch enable bits as selective fetch is turned off */ + bool clear_psr2_sel_fetch; bool req_psr2_sdp_prior_scanline; bool has_panel_replay; bool link_off_after_as_sdp_when_pr_active; diff --git a/drivers/gpu/drm/i915/display/intel_dp_link_caps.c b/drivers/gpu/drm/i915/display/intel_dp_link_caps.c index 98657aa4d3d5..abec3e2519ca 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_link_caps.c +++ b/drivers/gpu/drm/i915/display/intel_dp_link_caps.c @@ -3,6 +3,8 @@ * Copyright © 2026 Intel Corporation */ +#include + #include #include #include @@ -1302,14 +1304,14 @@ void intel_dp_link_caps_cleanup(struct intel_dp_link_caps *link_caps) const struct intel_dp_link_caps_test_ops i915_display_dp_link_caps_test_ops = { INTEL_DP_LINK_CAPS_TEST_OPS_INIT }; -EXPORT_SYMBOL(i915_display_dp_link_caps_test_ops); +EXPORT_SYMBOL_IF_KUNIT(i915_display_dp_link_caps_test_ops); #else const struct intel_dp_link_caps_test_ops intel_display_dp_link_caps_test_ops = { INTEL_DP_LINK_CAPS_TEST_OPS_INIT }; -EXPORT_SYMBOL(intel_display_dp_link_caps_test_ops); +EXPORT_SYMBOL_IF_KUNIT(intel_display_dp_link_caps_test_ops); #endif /* I915 */ diff --git a/drivers/gpu/drm/i915/display/intel_dp_link_training.c b/drivers/gpu/drm/i915/display/intel_dp_link_training.c index cb92cff90614..9a692f4fdfee 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_link_training.c +++ b/drivers/gpu/drm/i915/display/intel_dp_link_training.c @@ -2825,14 +2825,14 @@ void intel_dp_link_training_cleanup(struct intel_dp_link_training *link_training const struct intel_dp_link_training_test_ops i915_display_dp_link_training_test_ops = { INTEL_DP_LINK_TRAINING_TEST_OPS_INIT }; -EXPORT_SYMBOL(i915_display_dp_link_training_test_ops); +EXPORT_SYMBOL_IF_KUNIT(i915_display_dp_link_training_test_ops); #else const struct intel_dp_link_training_test_ops intel_display_dp_link_training_test_ops = { INTEL_DP_LINK_TRAINING_TEST_OPS_INIT }; -EXPORT_SYMBOL(intel_display_dp_link_training_test_ops); +EXPORT_SYMBOL_IF_KUNIT(intel_display_dp_link_training_test_ops); #endif /* I915 */ diff --git a/drivers/gpu/drm/i915/display/intel_dp_mst.c b/drivers/gpu/drm/i915/display/intel_dp_mst.c index 6a869d0f6ffc..0c362784afe4 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_mst.c +++ b/drivers/gpu/drm/i915/display/intel_dp_mst.c @@ -856,7 +856,8 @@ static u8 get_pipes_downstream_of_mst_port(struct intel_atomic_state *state, if (&connector->mst.dp->mst.mgr != mst_mgr) continue; - if (connector->mst.port != parent_port && + if (parent_port && + connector->mst.port != parent_port && !drm_dp_mst_port_downstream_of_parent(mst_mgr, connector->mst.port, parent_port)) @@ -2171,6 +2172,27 @@ bool intel_dp_mst_crtc_needs_modeset(struct intel_atomic_state *state, return false; } +bool intel_dp_mst_stream_disconnected(struct intel_atomic_state *state, + const struct intel_crtc *crtc) +{ + struct intel_connector *connector; + + connector = get_connector_in_state_for_crtc(state, crtc); + if (!connector) + return false; + + if (!connector->mst.dp) + return false; + + if (!connector->mst.dp->mst.mgr.mst_state) + return true; + + if (drm_connector_is_unregistered(&connector->base)) + return true; + + return false; +} + /** * intel_dp_mst_prepare_probe - Prepare an MST link for topology probing * @intel_dp: DP port object diff --git a/drivers/gpu/drm/i915/display/intel_dp_mst.h b/drivers/gpu/drm/i915/display/intel_dp_mst.h index ab09b487c6bb..8ce89242c05c 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_mst.h +++ b/drivers/gpu/drm/i915/display/intel_dp_mst.h @@ -28,6 +28,8 @@ int intel_dp_mst_atomic_check_link(struct intel_atomic_state *state, struct intel_link_bw_limits *limits); bool intel_dp_mst_crtc_needs_modeset(struct intel_atomic_state *state, struct intel_crtc *crtc); +bool intel_dp_mst_stream_disconnected(struct intel_atomic_state *state, + const struct intel_crtc *crtc); void intel_dp_mst_prepare_probe(struct intel_dp *intel_dp); bool intel_dp_mst_verify_dpcd_state(struct intel_dp *intel_dp); diff --git a/drivers/gpu/drm/i915/display/intel_link_bw.c b/drivers/gpu/drm/i915/display/intel_link_bw.c index b47474a3e9fe..e71e76d6fd3e 100644 --- a/drivers/gpu/drm/i915/display/intel_link_bw.c +++ b/drivers/gpu/drm/i915/display/intel_link_bw.c @@ -64,7 +64,8 @@ void intel_link_bw_init_limits(struct intel_atomic_state *state, intel_atomic_get_new_crtc_state(state, crtc); int forced_bpp_x16 = get_forced_link_bpp_x16(state, crtc); - if (state->base.duplicated && crtc_state) { + if ((state->base.duplicated && crtc_state) || + intel_dp_mst_stream_disconnected(state, crtc)) { limits->max_bpp_x16[pipe] = crtc_state->max_link_bpp_x16; if (intel_dsc_enabled_on_link(crtc_state)) limits->link_dsc_pipes |= BIT(pipe); diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c index f490beb66629..872e253db178 100644 --- a/drivers/gpu/drm/i915/display/intel_psr.c +++ b/drivers/gpu/drm/i915/display/intel_psr.c @@ -2889,6 +2889,8 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, struct intel_crtc *crtc) { struct intel_display *display = to_intel_display(state); + const struct intel_crtc_state *old_crtc_state = + intel_atomic_get_old_crtc_state(state, crtc); struct intel_crtc_state *crtc_state = intel_atomic_get_new_crtc_state(state, crtc); struct intel_plane_state *new_plane_state, *old_plane_state; struct intel_plane *plane; @@ -2901,6 +2903,19 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, bool full_update = false, su_area_changed; int i, ret; + /* + * Selective fetch is not always usable, for instance it is dropped + * while pipe CRC is active. The planes keep their selective fetch + * enable bit set in hardware over that, and a plane disabled while + * selective fetch is off never gets the bit cleared. Once selective + * fetch comes back the hardware would resume fetching for a plane that + * is no longer enabled and keep its DDB range reserved, so have the + * plane update drop the bit for every plane of the pipe as selective + * fetch is turned off. + */ + crtc_state->clear_psr2_sel_fetch = old_crtc_state->enable_psr2_sel_fetch && + !crtc_state->enable_psr2_sel_fetch; + if (!crtc_state->enable_psr2_sel_fetch) return 0; diff --git a/drivers/gpu/drm/i915/display/intel_quirks.c b/drivers/gpu/drm/i915/display/intel_quirks.c index 33245f44c0d5..7d7db774d8c7 100644 --- a/drivers/gpu/drm/i915/display/intel_quirks.c +++ b/drivers/gpu/drm/i915/display/intel_quirks.c @@ -257,6 +257,9 @@ static struct intel_quirk intel_quirks[] = { /* Dell XPS 13 7390 2-in-1 */ { 0x8a52, 0x1028, 0x08b0, quirk_edp_limit_rate_hbr2 }, + /* HP Pavilion Plus Laptop 14-ew1xxx */ + { 0x7d55, 0x103c, 0x8c31, quirk_edp_limit_rate_hbr2 }, + /* Xiaomi Book Pro 14 2026 */ { 0xb081, 0x1d72, 0x2424, quirk_disable_psr2 }, }; diff --git a/drivers/gpu/drm/i915/display/intel_vrr.c b/drivers/gpu/drm/i915/display/intel_vrr.c index e36db1174440..1698c54e258b 100644 --- a/drivers/gpu/drm/i915/display/intel_vrr.c +++ b/drivers/gpu/drm/i915/display/intel_vrr.c @@ -439,10 +439,12 @@ static bool intel_vrr_dc_balance_possible(const struct intel_crtc_state *crtc_st static void intel_vrr_dc_balance_compute_config(struct intel_crtc_state *crtc_state) { + struct intel_display *display = to_intel_display(crtc_state); int guardband_usec, adjustment_usec; struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode; - if (!intel_vrr_dc_balance_possible(crtc_state) || !crtc_state->vrr.enable) + if (!intel_vrr_dc_balance_possible(crtc_state) || + !crtc_state->vrr.enable || !display->params.enable_dc_balance) return; crtc_state->vrr.dc_balance.vmax = crtc_state->vrr.vmax; diff --git a/drivers/gpu/drm/i915/display/skl_universal_plane.c b/drivers/gpu/drm/i915/display/skl_universal_plane.c index 45e8379d93d4..271c0640fd36 100644 --- a/drivers/gpu/drm/i915/display/skl_universal_plane.c +++ b/drivers/gpu/drm/i915/display/skl_universal_plane.c @@ -886,7 +886,8 @@ static void icl_plane_disable_sel_fetch_arm(struct intel_dsb *dsb, struct intel_display *display = to_intel_display(plane); enum pipe pipe = plane->pipe; - if (!crtc_state->enable_psr2_sel_fetch) + if (!crtc_state->enable_psr2_sel_fetch && + !crtc_state->clear_psr2_sel_fetch) return; intel_de_write_dsb(display, dsb, SEL_FETCH_PLANE_CTL(pipe, plane->id), 0); @@ -1635,10 +1636,8 @@ static void icl_plane_update_sel_fetch_arm(struct intel_dsb *dsb, struct intel_display *display = to_intel_display(plane); enum pipe pipe = plane->pipe; - if (!crtc_state->enable_psr2_sel_fetch) - return; - - if (drm_rect_height(&plane_state->psr2_sel_fetch_area) > 0) + if (crtc_state->enable_psr2_sel_fetch && + drm_rect_height(&plane_state->psr2_sel_fetch_area) > 0) intel_de_write_dsb(display, dsb, SEL_FETCH_PLANE_CTL(pipe, plane->id), SEL_FETCH_PLANE_CTL_ENABLE); else diff --git a/drivers/gpu/drm/imagination/pvr_free_list.c b/drivers/gpu/drm/imagination/pvr_free_list.c index e85cac83834c..faf5e586d8dc 100644 --- a/drivers/gpu/drm/imagination/pvr_free_list.c +++ b/drivers/gpu/drm/imagination/pvr_free_list.c @@ -8,6 +8,7 @@ #include "pvr_vm.h" #include +#include #include #include #include @@ -612,13 +613,21 @@ pvr_free_list_process_reconstruct_req(struct pvr_device *pvr_dev, }; struct rogue_fwif_freelists_reconstruction_data *resp = &resp_cmd.cmd_data.free_lists_reconstruction_data; + u32 count = min_t(u32, req->freelist_count, + ARRAY_SIZE(req->freelist_ids)); - for (u32 i = 0; i < req->freelist_count; i++) + if (count != req->freelist_count) { + drm_warn_once(from_pvr_device(pvr_dev), + "Requested reconstruction of %u freelists, limiting to %u\n", + req->freelist_count, count); + } + + for (u32 i = 0; i < count; i++) pvr_free_list_reconstruct(pvr_dev, req->freelist_ids[i]); - resp->freelist_count = req->freelist_count; + resp->freelist_count = count; memcpy(resp->freelist_ids, req->freelist_ids, - req->freelist_count * sizeof(resp->freelist_ids[0])); + count * sizeof(resp->freelist_ids[0])); WARN_ON(pvr_kccb_send_cmd(pvr_dev, &resp_cmd, NULL)); } diff --git a/drivers/gpu/drm/imagination/pvr_mmu.c b/drivers/gpu/drm/imagination/pvr_mmu.c index 3cac482e1034..62eae7fcd5a2 100644 --- a/drivers/gpu/drm/imagination/pvr_mmu.c +++ b/drivers/gpu/drm/imagination/pvr_mmu.c @@ -12,6 +12,7 @@ #include "pvr_rogue_mmu_defs.h" #include +#include #include #include #include @@ -2335,6 +2336,7 @@ void pvr_mmu_op_context_destroy(struct pvr_mmu_op_context *op_ctx) * pvr_mmu_op_context_create() - Create an MMU op context. * @ctx: MMU context associated with owning VM context. * @sgt: Scatter gather table containing pages pinned for use by this context. + * @device_addr: Virtual device address at the start of the requested mapping. * @sgt_offset: Start offset of the requested device-virtual memory mapping. * @size: Size in bytes of the requested device-virtual memory mapping. For an * unmapping, this should be zero so that no page tables are allocated. @@ -2346,8 +2348,9 @@ void pvr_mmu_op_context_destroy(struct pvr_mmu_op_context *op_ctx) */ struct pvr_mmu_op_context * pvr_mmu_op_context_create(struct pvr_mmu_context *ctx, struct sg_table *sgt, - u64 sgt_offset, u64 size) + u64 device_addr, u64 sgt_offset, u64 size) { + u64 start_addr = device_addr + sgt_offset; int err; struct pvr_mmu_op_context *op_ctx = kzalloc_obj(*op_ctx); @@ -2363,16 +2366,16 @@ pvr_mmu_op_context_create(struct pvr_mmu_context *ctx, struct sg_table *sgt, if (size) { /* * The number of page table objects we need to prealloc is - * indicated by the mapping size, start offset and the sizes + * indicated by the mapping size, start address and the sizes * of the areas mapped per PT or PD. The range calculation is * identical to that for the index into a table for a device * address, so we reuse those functions here. */ - const u32 l1_start_idx = pvr_page_table_l2_idx(sgt_offset); - const u32 l1_end_idx = pvr_page_table_l2_idx(sgt_offset + size); + const u32 l1_start_idx = pvr_page_table_l2_idx(start_addr); + const u32 l1_end_idx = pvr_page_table_l2_idx(start_addr + size); const u32 l1_count = l1_end_idx - l1_start_idx + 1; - const u32 l0_start_idx = pvr_page_table_l1_idx(sgt_offset); - const u32 l0_end_idx = pvr_page_table_l1_idx(sgt_offset + size); + const u32 l0_start_idx = pvr_page_table_l1_idx(start_addr); + const u32 l0_end_idx = pvr_page_table_l1_idx(start_addr + size); const u32 l0_count = l0_end_idx - l0_start_idx + 1; /* @@ -2553,7 +2556,9 @@ pvr_mmu_map_sgl(struct pvr_mmu_op_context *op_ctx, struct scatterlist *sgl, err_destroy_pages: memcpy(&op_ctx->curr_page, &ptr_copy, sizeof(op_ctx->curr_page)); - err = pvr_mmu_op_context_unmap_curr_page(op_ctx, page); + if (pvr_mmu_op_context_unmap_curr_page(op_ctx, page)) + drm_err(from_pvr_device(op_ctx->mmu_ctx->pvr_dev), + "%s : Failure in unmapping pages\n", __func__); return err; } diff --git a/drivers/gpu/drm/imagination/pvr_mmu.h b/drivers/gpu/drm/imagination/pvr_mmu.h index a8ecd460168d..2c02d61ba0a2 100644 --- a/drivers/gpu/drm/imagination/pvr_mmu.h +++ b/drivers/gpu/drm/imagination/pvr_mmu.h @@ -99,7 +99,7 @@ dma_addr_t pvr_mmu_get_root_table_dma_addr(struct pvr_mmu_context *ctx); void pvr_mmu_op_context_destroy(struct pvr_mmu_op_context *op_ctx); struct pvr_mmu_op_context * pvr_mmu_op_context_create(struct pvr_mmu_context *ctx, - struct sg_table *sgt, u64 sgt_offset, u64 size); + struct sg_table *sgt, u64 device_addr, u64 sgt_offset, u64 size); int pvr_mmu_map(struct pvr_mmu_op_context *op_ctx, u64 size, u64 flags, u64 device_addr); diff --git a/drivers/gpu/drm/imagination/pvr_vm.c b/drivers/gpu/drm/imagination/pvr_vm.c index ceb78694cd98..55cc999f3708 100644 --- a/drivers/gpu/drm/imagination/pvr_vm.c +++ b/drivers/gpu/drm/imagination/pvr_vm.c @@ -276,7 +276,7 @@ pvr_vm_bind_op_map_init(struct pvr_vm_bind_op *bind_op, goto err_bind_op_fini; bind_op->mmu_op_ctx = - pvr_mmu_op_context_create(vm_ctx->mmu_ctx, sgt, offset, size); + pvr_mmu_op_context_create(vm_ctx->mmu_ctx, sgt, device_addr, offset, size); err = PTR_ERR_OR_ZERO(bind_op->mmu_op_ctx); if (err) { bind_op->mmu_op_ctx = NULL; @@ -318,7 +318,7 @@ pvr_vm_bind_op_unmap_init(struct pvr_vm_bind_op *bind_op, } bind_op->mmu_op_ctx = - pvr_mmu_op_context_create(vm_ctx->mmu_ctx, NULL, 0, 0); + pvr_mmu_op_context_create(vm_ctx->mmu_ctx, NULL, device_addr, 0, 0); err = PTR_ERR_OR_ZERO(bind_op->mmu_op_ctx); if (err) { bind_op->mmu_op_ctx = NULL; diff --git a/drivers/gpu/drm/mediatek/mtk_drm_drv.c b/drivers/gpu/drm/mediatek/mtk_drm_drv.c index c86a3f54f35b..dc8e3e917dd7 100644 --- a/drivers/gpu/drm/mediatek/mtk_drm_drv.c +++ b/drivers/gpu/drm/mediatek/mtk_drm_drv.c @@ -638,8 +638,10 @@ static int mtk_drm_bind(struct device *dev) private->mtk_drm_bound = true; private->dev = dev; - if (!mtk_drm_get_all_drm_priv(dev)) + if (!mtk_drm_get_all_drm_priv(dev)) { + put_device(private->mutex_dev); return 0; + } drm = drm_dev_alloc(&mtk_drm_driver, dev); if (IS_ERR(drm)) { @@ -1060,7 +1062,7 @@ static int mtk_drm_probe(struct platform_device *pdev) struct mtk_mmsys_driver_data *mtk_drm_data; struct device_node *node; struct component_match *match = NULL; - struct platform_device *ovl_adaptor; + struct platform_device *ovl_adaptor = NULL; int ret; int i; @@ -1112,6 +1114,9 @@ static int mtk_drm_probe(struct platform_device *pdev) PLATFORM_DEVID_AUTO, (void *)private->mmsys_dev, sizeof(*private->mmsys_dev)); + if (IS_ERR(ovl_adaptor)) + return PTR_ERR(ovl_adaptor); + private->ddp_comp[DDP_COMPONENT_DRM_OVL_ADAPTOR].dev = &ovl_adaptor->dev; mtk_ddp_comp_init(dev, NULL, &private->ddp_comp[DDP_COMPONENT_DRM_OVL_ADAPTOR], DDP_COMPONENT_DRM_OVL_ADAPTOR); @@ -1205,6 +1210,8 @@ static int mtk_drm_probe(struct platform_device *pdev) err_pm: pm_runtime_disable(dev); err_node: + if (ovl_adaptor) + platform_device_unregister(ovl_adaptor); of_node_put(private->mutex_node); for (i = 0; i < DDP_COMPONENT_DRM_ID_MAX; i++) of_node_put(private->comp_node[i]); @@ -1214,9 +1221,15 @@ static int mtk_drm_probe(struct platform_device *pdev) static void mtk_drm_remove(struct platform_device *pdev) { struct mtk_drm_private *private = platform_get_drvdata(pdev); + struct device *ovl_adaptor_dev; int i; + ovl_adaptor_dev = + private->ddp_comp[DDP_COMPONENT_DRM_OVL_ADAPTOR].dev; + component_master_del(&pdev->dev, &mtk_drm_ops); + if (ovl_adaptor_dev) + platform_device_unregister(to_platform_device(ovl_adaptor_dev)); pm_runtime_disable(&pdev->dev); of_node_put(private->mutex_node); for (i = 0; i < DDP_COMPONENT_DRM_ID_MAX; i++) diff --git a/drivers/gpu/drm/mediatek/mtk_hdmi_ddc_v2.c b/drivers/gpu/drm/mediatek/mtk_hdmi_ddc_v2.c index 31e81a6de6d8..1ec973d3f52c 100644 --- a/drivers/gpu/drm/mediatek/mtk_hdmi_ddc_v2.c +++ b/drivers/gpu/drm/mediatek/mtk_hdmi_ddc_v2.c @@ -376,8 +376,10 @@ static int mtk_hdmi_ddc_v2_probe(struct platform_device *pdev) pm_runtime_get_sync(dev); ret = devm_i2c_add_adapter(dev, &ddc->adap); - if (ret < 0) + if (ret < 0) { + pm_runtime_put_noidle(dev); return dev_err_probe(dev, ret, "Cannot add DDC I2C adapter\n"); + } platform_set_drvdata(pdev, ddc); return 0; diff --git a/drivers/gpu/drm/mediatek/mtk_hdmi_regs_v2.h b/drivers/gpu/drm/mediatek/mtk_hdmi_regs_v2.h index 521b35c7e14d..bf3ebf5492d1 100644 --- a/drivers/gpu/drm/mediatek/mtk_hdmi_regs_v2.h +++ b/drivers/gpu/drm/mediatek/mtk_hdmi_regs_v2.h @@ -173,7 +173,7 @@ #define TPI_AUDIO_LOOKUP_EN BIT(2) /* Video downsampling configuration */ -#define VID_DOWNSAMPLE_CONFIG 0x8d0 +#define VID_DOWNSAMPLE_CONFIG 0x8f0 #define C444_C422_CONFIG_ENABLE BIT(0) #define C422_C420_CONFIG_ENABLE BIT(4) #define C422_C420_CONFIG_BYPASS BIT(5) diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl0080.h b/drivers/gpu/drm/nouveau/include/nvif/cl0080.h index ea8267e0d8da..9e639df1da46 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/cl0080.h +++ b/drivers/gpu/drm/nouveau/include/nvif/cl0080.h @@ -4,6 +4,7 @@ #define NV_DEVICE_V0_INFO 0x00 #define NV_DEVICE_V0_TIME 0x01 +#define NV_DEVICE_V0_GCX_READY 0x02 struct nv_device_info_v0 { __u8 version; @@ -55,6 +56,15 @@ struct nv_device_time_v0 { __u64 time; }; +#define NV_DEVICE_GC6_READY BIT(0) +#define NV_DEVICE_GCOFF_READY BIT(1) + +struct nv_device_gcx_ready_v0 { + __u8 version; + __u8 pad01[6]; + __u8 ready; +}; + #define NV_DEVICE_INFO_UNIT (0xffffffffULL << 32) #define NV_DEVICE_INFO(n) ((n) | (0x00000000ULL << 32)) #define NV_DEVICE_HOST(n) ((n) | (0x00000001ULL << 32)) diff --git a/drivers/gpu/drm/nouveau/include/nvif/device.h b/drivers/gpu/drm/nouveau/include/nvif/device.h index 7877a2a79da9..ce2fadcb05d9 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/device.h +++ b/drivers/gpu/drm/nouveau/include/nvif/device.h @@ -22,4 +22,5 @@ int nvif_device_ctor(struct nvif_client *, const char *name, struct nvif_device void nvif_device_dtor(struct nvif_device *); int nvif_device_map(struct nvif_device *); u64 nvif_device_time(struct nvif_device *); +int nvif_device_gcx_ready(struct nvif_device *); #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h index 64fed208e4cf..ed5c6e0e68d3 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h @@ -156,6 +156,10 @@ struct nvkm_gsp { struct sg_table fbsr; } sr; + struct { + bool use_raw_mode_comptagline_alloc; + } memsys; + struct { struct nvkm_gsp_mem mem; @@ -495,6 +499,8 @@ nvkm_gsp_event_dtor(struct nvkm_gsp_event *event) int nvkm_gsp_intr_stall(struct nvkm_gsp *, enum nvkm_subdev_type, int); int nvkm_gsp_intr_nonstall(struct nvkm_gsp *, enum nvkm_subdev_type, int); +int nvkm_gsp_gcx_ready(struct nvkm_gsp *gsp); + int gv100_gsp_new(struct nvkm_device *, enum nvkm_subdev_type, int, struct nvkm_gsp **); int tu102_gsp_new(struct nvkm_device *, enum nvkm_subdev_type, int, struct nvkm_gsp **); int tu116_gsp_new(struct nvkm_device *, enum nvkm_subdev_type, int, struct nvkm_gsp **); diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index 0e8de6d4b36f..6dcb92575eb4 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -578,8 +578,9 @@ int nouveau_bo_pin_locked(struct nouveau_bo *nvbo, uint32_t domain, bool contig) "0x%08x vs 0x%08x\n", bo, bo->resource->mem_type, domain); ret = -EBUSY; + } else { + ttm_bo_pin(&nvbo->bo); } - ttm_bo_pin(&nvbo->bo); goto out; } diff --git a/drivers/gpu/drm/nouveau/nouveau_dmem.c b/drivers/gpu/drm/nouveau/nouveau_dmem.c index ad4570c50be7..e74d7bb975a8 100644 --- a/drivers/gpu/drm/nouveau/nouveau_dmem.c +++ b/drivers/gpu/drm/nouveau/nouveau_dmem.c @@ -339,8 +339,8 @@ nouveau_dmem_chunk_alloc(struct nouveau_drm *drm, struct page **ppage, chunk->pagemap.ops = &nouveau_dmem_pagemap_ops; chunk->pagemap.owner = drm->dev; - ret = nouveau_bo_new_pin(&drm->client, NOUVEAU_GEM_DOMAIN_VRAM, DMEM_CHUNK_SIZE, - &chunk->bo); + ret = nouveau_bo_new_pin(&drm->client, NOUVEAU_GEM_DOMAIN_VRAM, + DMEM_CHUNK_SIZE * NR_CHUNKS, &chunk->bo); if (ret) goto out_release; diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c index b0f9fb10a74d..5e1336615cf6 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drm.c +++ b/drivers/gpu/drm/nouveau/nouveau_drm.c @@ -585,6 +585,7 @@ nouveau_drm_device_fini(struct nouveau_drm *drm) if (nouveau_pmops_runtime()) { pm_runtime_get_sync(dev->dev); pm_runtime_forbid(dev->dev); + pm_runtime_dont_use_autosuspend(dev->dev); } nouveau_led_fini(dev); @@ -1157,6 +1158,7 @@ nouveau_pmops_runtime_suspend(struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); struct nouveau_drm *drm = pci_get_drvdata(pdev); + struct nvif_device *nvif = &drm->client.device; int ret; if (!nouveau_pmops_runtime()) { @@ -1164,6 +1166,18 @@ nouveau_pmops_runtime_suspend(struct device *dev) return -EBUSY; } + // Check if the GPU itself is ready for runtime suspend, otherwise mark as busy and check + // again in a bit. + ret = nvif_device_gcx_ready(nvif); + if (ret < 0) { + NV_ERROR(drm, "Failed to query GCX readiness (returned %d)\n", ret); + return -EBUSY; + } else if (!(ret & NV_DEVICE_GCOFF_READY)) { + NV_DEBUG(drm, "GPU isn't ready for suspend yet, delaying...\n"); + pm_runtime_mark_last_busy(dev); + return -EBUSY; + } + nouveau_switcheroo_optimus_dsm(); ret = nouveau_do_suspend(drm, true); pci_save_state(pdev); @@ -1259,10 +1273,8 @@ nouveau_drm_open(struct drm_device *dev, struct drm_file *fpriv) mutex_unlock(&drm->clients_lock); done: - if (ret && cli) { - nouveau_cli_fini(cli); + if (ret && cli) kfree(cli); - } pm_runtime_mark_last_busy(dev->dev); pm_runtime_put_autosuspend(dev->dev); diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c index 0b7123b163e4..51188be57221 100644 --- a/drivers/gpu/drm/nouveau/nouveau_gem.c +++ b/drivers/gpu/drm/nouveau/nouveau_gem.c @@ -522,6 +522,7 @@ validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, if (unlikely(ret)) { if (ret != -ERESTARTSYS) NV_PRINTK(err, cli, "fail reserve\n"); + drm_gem_object_put(gem); break; } } @@ -531,6 +532,7 @@ validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm); if (!vma) { NV_PRINTK(err, cli, "vma not found!\n"); + drm_gem_object_put(gem); ret = -EINVAL; break; } diff --git a/drivers/gpu/drm/nouveau/nouveau_sched.c b/drivers/gpu/drm/nouveau/nouveau_sched.c index 8b9f935afe09..b3f02c490ecb 100644 --- a/drivers/gpu/drm/nouveau/nouveau_sched.c +++ b/drivers/gpu/drm/nouveau/nouveau_sched.c @@ -517,7 +517,7 @@ nouveau_sched_destroy(struct nouveau_sched **psched) struct nouveau_sched *sched = *psched; nouveau_sched_fini(sched); - kfree(sched); + kfree_rcu(sched, rcu); *psched = NULL; } diff --git a/drivers/gpu/drm/nouveau/nouveau_sched.h b/drivers/gpu/drm/nouveau/nouveau_sched.h index 20cd1da8db73..51ce8dcf6285 100644 --- a/drivers/gpu/drm/nouveau/nouveau_sched.h +++ b/drivers/gpu/drm/nouveau/nouveau_sched.h @@ -98,6 +98,7 @@ void nouveau_job_free(struct nouveau_job *job); struct nouveau_sched { struct drm_gpu_scheduler base; + struct rcu_head rcu; struct drm_sched_entity entity; struct workqueue_struct *wq; struct mutex mutex; diff --git a/drivers/gpu/drm/nouveau/nouveau_uvmm.c b/drivers/gpu/drm/nouveau/nouveau_uvmm.c index fc125fd44a9b..2026fe6b48c6 100644 --- a/drivers/gpu/drm/nouveau/nouveau_uvmm.c +++ b/drivers/gpu/drm/nouveau/nouveau_uvmm.c @@ -846,6 +846,9 @@ op_map(struct nouveau_uvma *uvma) { struct nouveau_bo *nvbo = nouveau_gem_object(uvma->va.gem.obj); + if (drm_gpuva_invalidated(&uvma->va)) + return; + nouveau_uvma_map(uvma, nouveau_mem(nvbo->bo.resource)); } @@ -1232,6 +1235,7 @@ bind_lock_validate(struct nouveau_job *job, struct drm_exec *exec, drm_gpuva_for_each_op(va_op, op->ops) { struct drm_gem_object *obj = op_gem_obj(va_op); + struct nouveau_bo *nvbo; if (unlikely(!obj)) continue; @@ -1246,8 +1250,13 @@ bind_lock_validate(struct nouveau_job *job, struct drm_exec *exec, if (va_op->op == DRM_GPUVA_OP_UNMAP) continue; - ret = nouveau_bo_validate(nouveau_gem_object(obj), - true, false); + nvbo = nouveau_gem_object(obj); + if (!(nvbo->valid_domains & + (NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART))) + return -EINVAL; + + nouveau_bo_placement_set(nvbo, nvbo->valid_domains, 0); + ret = nouveau_bo_validate(nvbo, true, false); if (ret) return ret; } diff --git a/drivers/gpu/drm/nouveau/nvif/device.c b/drivers/gpu/drm/nouveau/nvif/device.c index 24880931039f..1be9fbe6cb70 100644 --- a/drivers/gpu/drm/nouveau/nvif/device.c +++ b/drivers/gpu/drm/nouveau/nvif/device.c @@ -38,6 +38,19 @@ nvif_device_time(struct nvif_device *device) return device->user.func->time(&device->user); } +int +nvif_device_gcx_ready(struct nvif_device *device) +{ + struct nv_device_gcx_ready_v0 args = {}; + int ret; + + ret = nvif_object_mthd(&device->object, NV_DEVICE_V0_GCX_READY, &args, sizeof(args)); + if (ret) + return ret; + + return args.ready; +} + int nvif_device_map(struct nvif_device *device) { diff --git a/drivers/gpu/drm/nouveau/nvif/vmm.c b/drivers/gpu/drm/nouveau/nvif/vmm.c index 65c3e883b119..579af70766f2 100644 --- a/drivers/gpu/drm/nouveau/nvif/vmm.c +++ b/drivers/gpu/drm/nouveau/nvif/vmm.c @@ -192,6 +192,7 @@ void nvif_vmm_dtor(struct nvif_vmm *vmm) { kfree(vmm->page); + vmm->page = NULL; nvif_object_dtor(&vmm->object); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/ctrl.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/ctrl.c index f2e9a06263ce..28702741a88b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/ctrl.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/ctrl.c @@ -74,6 +74,7 @@ nvkm_control_mthd_pstate_attr(struct nvkm_control *ctrl, void *data, u32 size) const struct nvkm_domain *domain; struct nvkm_pstate *pstate; struct nvkm_cstate *cstate; + bool found = false; int i = 0, j = -1; u32 lo, hi; int ret = -ENOSYS; @@ -104,10 +105,15 @@ nvkm_control_mthd_pstate_attr(struct nvkm_control *ctrl, void *data, u32 size) if (args->v0.state != NVIF_CONTROL_PSTATE_ATTR_V0_STATE_CURRENT) { list_for_each_entry(pstate, &clk->states, head) { - if (i++ == args->v0.state) + if (i++ == args->v0.state) { + found = true; break; + } } + if (!found) + return -EINVAL; + lo = pstate->base.domain[domain->name]; hi = lo; list_for_each_entry(cstate, &pstate->list, head) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c index 23d11d8221cb..f78e6b9b4292 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c @@ -27,6 +27,7 @@ #include #include +#include #include #include @@ -189,6 +190,38 @@ nvkm_udevice_time(struct nvkm_udevice *udev, void *data, u32 size) return ret; } +static int +nvkm_udevice_gcx_ready(struct nvkm_udevice *udev, void *data, u32 size) +{ + struct nvkm_object *object = &udev->object; + struct nvkm_device *device = udev->device; + struct nvkm_gsp *gsp = device->gsp; + union { + struct nv_device_gcx_ready_v0 v0; + } *args = data; + int ret = -ENOSYS; + + if (!gsp) { + args->v0.ready = NV_DEVICE_GC6_READY | NV_DEVICE_GCOFF_READY; + return 0; + } + + nvif_ioctl(object, "device gcx ready size %d\n", size); + ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false); + if (!ret) { + nvif_ioctl(object, "device gcx ready vers %d\n", args->v0.version); + + ret = nvkm_gsp_gcx_ready(gsp); + if (ret < 0) + return ret; + + args->v0.ready = ret; + ret = 0; + } + + return ret; +} + static int nvkm_udevice_mthd(struct nvkm_object *object, u32 mthd, void *data, u32 size) { @@ -199,6 +232,8 @@ nvkm_udevice_mthd(struct nvkm_object *object, u32 mthd, void *data, u32 size) return nvkm_udevice_info(udev, data, size); case NV_DEVICE_V0_TIME: return nvkm_udevice_time(udev, data, size); + case NV_DEVICE_V0_GCX_READY: + return nvkm_udevice_gcx_ready(udev, data, size); default: break; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c index 377d0e0cef84..9887b3898505 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c @@ -253,8 +253,7 @@ nvkm_uoutp_mthd_hdmi(struct nvkm_outp *outp, void *argv, u32 argc) if (!ior->func->hdmi || args->v0.max_ac_packet > 0x1f || - args->v0.rekey > 0x7f || - (args->v0.scdc && !ior->func->hdmi->scdc)) + args->v0.rekey > 0x7f) return -EINVAL; if (!args->v0.enable) { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c index 572e63846315..1cb83edc78dc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/base.c @@ -199,16 +199,18 @@ nvkm_cstate_prog(struct nvkm_clk *clk, struct nvkm_pstate *pstate, int cstatei) } if (volt) { - ret = nvkm_volt_set_id(volt, cstate->voltage, - pstate->base.voltage, clk->temp, -1); - if (ret && ret != -ENODEV) - nvkm_error(subdev, "failed to lower voltage: %d\n", ret); + int err = nvkm_volt_set_id(volt, cstate->voltage, + pstate->base.voltage, clk->temp, -1); + + if (err && err != -ENODEV) + nvkm_error(subdev, "failed to lower voltage: %d\n", err); } if (therm) { - ret = nvkm_therm_cstate(therm, pstate->fanspeed, -1); - if (ret && ret != -ENODEV) - nvkm_error(subdev, "failed to lower fan speed: %d\n", ret); + int err = nvkm_therm_cstate(therm, pstate->fanspeed, -1); + + if (err && err != -ENODEV) + nvkm_error(subdev, "failed to lower fan speed: %d\n", err); } return ret; @@ -270,13 +272,19 @@ nvkm_pstate_prog(struct nvkm_clk *clk, int pstatei) struct nvkm_fb *fb = subdev->device->fb; struct nvkm_pci *pci = subdev->device->pci; struct nvkm_pstate *pstate; + bool found = false; int ret, idx = 0; list_for_each_entry(pstate, &clk->states, head) { - if (idx++ == pstatei) + if (idx++ == pstatei) { + found = true; break; + } } + if (!found) + return -EINVAL; + nvkm_debug(subdev, "setting performance state %d\n", pstatei); clk->pstate = pstatei; @@ -473,6 +481,7 @@ static int nvkm_clk_ustate_update(struct nvkm_clk *clk, int req) { struct nvkm_pstate *pstate; + bool found = false; int i = 0; if (!clk->allow_reclock) @@ -480,12 +489,14 @@ nvkm_clk_ustate_update(struct nvkm_clk *clk, int req) if (req != -1 && req != -2) { list_for_each_entry(pstate, &clk->states, head) { - if (pstate->pstate == req) + if (pstate->pstate == req) { + found = true; break; + } i++; } - if (pstate->pstate != req) + if (!found) return -EINVAL; req = i; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c index 18241c6ba5fa..4d52a158f320 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c @@ -51,6 +51,8 @@ nv1a_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram) mib = ((mem >> 4) & 127) + 1; } + pci_dev_put(bridge); + return nvkm_ram_new_(&nv04_ram_func, fb, NVKM_RAM_TYPE_STOLEN, mib * 1024 * 1024, pram); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/base.c index 9ba1316831e7..e475d0e8fa7b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/base.c @@ -20,6 +20,7 @@ * OTHER DEALINGS IN THE SOFTWARE. */ #include "priv.h" +#include int nvkm_gsp_intr_nonstall(struct nvkm_gsp *gsp, enum nvkm_subdev_type type, int inst) @@ -47,6 +48,15 @@ nvkm_gsp_intr_stall(struct nvkm_gsp *gsp, enum nvkm_subdev_type type, int inst) return -ENOENT; } +int +nvkm_gsp_gcx_ready(struct nvkm_gsp *gsp) +{ + if (!gsp->rm->api->gsp->gcx_ready) + return NV_DEVICE_GC6_READY | NV_DEVICE_GCOFF_READY; + + return gsp->rm->api->gsp->gcx_ready(gsp); +} + static int nvkm_gsp_fini(struct nvkm_subdev *subdev, enum nvkm_suspend_state suspend) { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/priv.h index 71b7203bef50..b07797813b04 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/priv.h @@ -93,6 +93,7 @@ void r535_gsp_dtor(struct nvkm_gsp *); int r535_gsp_oneinit(struct nvkm_gsp *); int r535_gsp_init(struct nvkm_gsp *); int r535_gsp_fini(struct nvkm_gsp *, enum nvkm_suspend_state suspend); +int r535_gsp_gcx_ready(struct nvkm_gsp *gsp); int nvkm_gsp_new_(const struct nvkm_gsp_fwif *, struct nvkm_device *, enum nvkm_subdev_type, int, struct nvkm_gsp **); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gsp.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gsp.c index 94925f1590ea..63aa30f94747 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gsp.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/gsp.c @@ -1782,6 +1782,23 @@ r535_gsp_fini(struct nvkm_gsp *gsp, enum nvkm_suspend_state suspend) return 0; } +int +r535_gsp_get_static_memsys_info(struct nvkm_gsp *gsp) +{ + NV2080_CTRL_INTERNAL_MEMSYS_GET_STATIC_CONFIG_PARAMS *ctrl; + + ctrl = nvkm_gsp_rm_ctrl_rd(&gsp->internal.device.subdevice, + NV2080_CTRL_CMD_INTERNAL_MEMSYS_GET_STATIC_CONFIG, + sizeof(*ctrl)); + if (IS_ERR(ctrl)) + return PTR_ERR(ctrl); + + gsp->memsys.use_raw_mode_comptagline_alloc = ctrl->bUseRawModeComptaglineAllocation; + + nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl); + return 0; +} + int r535_gsp_init(struct nvkm_gsp *gsp) { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/nvrm/gsp.h b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/nvrm/gsp.h index b6683a5bf870..7b10b7548c57 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/nvrm/gsp.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/nvrm/gsp.h @@ -782,6 +782,51 @@ typedef struct NV2080_CTRL_INTERNAL_INTR_GET_KERNEL_TABLE_PARAMS { #define GSP_FW_HEAP_PARAM_CLIENT_ALLOC_SIZE ((48 << 10) * 2048) // Support 2048 channels +typedef struct NV2080_CTRL_INTERNAL_MEMSYS_GET_STATIC_CONFIG_PARAMS { + /*! Determines if RM should use 1 to 1 Comptagline allocation policy */ + NvBool bOneToOneComptagLineAllocation; + + /*! Determines if RM should use 1 to 4 Comptagline allocation policy */ + NvBool bUseOneToFourComptagLineAllocation; + + /*! Determines if RM should use raw Comptagline allocation policy */ + NvBool bUseRawModeComptaglineAllocation; + + /*! Has COMPBIT_BACKING_SIZE been overridden to zero (i.e. disabled)? */ + NvBool bDisableCompbitBacking; + + /*! Determine if we need to disable post L2 compression */ + NvBool bDisablePostL2Compression; + + /*! Is ECC DRAM feature supported? */ + NvBool bEnabledEccFBPA; + + NvBool bL2PreFill; + + /*! L2 cache size */ + NV_DECLARE_ALIGNED(NvU64 l2CacheSize, 8); + + /*! Indicate whether fpba is present or not */ + NvBool bFbpaPresent; + + /*! Size covered by one comptag */ + NvU32 comprPageSize; + + /*! log32(comprPageSize) */ + NvU32 comprPageShift; + + /*! RAM type */ + NvU32 ramType; + + /*! LTC count */ + NvU32 ltcCount; + + /*! LTS per LTC count */ + NvU32 ltsPerLtcCount; +} NV2080_CTRL_INTERNAL_MEMSYS_GET_STATIC_CONFIG_PARAMS; + +#define NV2080_CTRL_CMD_INTERNAL_MEMSYS_GET_STATIC_CONFIG (0x20800a1c) /* finn: Evaluated from "(FINN_NV20_SUBDEVICE_0_INTERNAL_INTERFACE_ID << 8) | NV2080_CTRL_INTERNAL_MEMSYS_GET_STATIC_CONFIG_PARAMS_MESSAGE_ID" */ + typedef union rpc_message_rpc_union_field_v03_00 { NvU32 spare; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/fbsr.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/fbsr.c index af5aa5065c3d..469e7eed1d6f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/fbsr.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/fbsr.c @@ -26,6 +26,52 @@ r570_fbsr_suspend_channels(struct nvkm_gsp *gsp, bool suspend) return nvkm_gsp_rm_ctrl_wr(&gsp->internal.device.subdevice, ctrl); } +static int +r570_fb_get_compbit_store_size(struct nvkm_gsp *gsp, u64 *size) +{ + NV0080_CTRL_FB_GET_COMPBIT_STORE_INFO_PARAMS *ctrl; + + ctrl = nvkm_gsp_rm_ctrl_rd(&gsp->internal.device.object, + NV0080_CTRL_CMD_FB_GET_COMPBIT_STORE_INFO, + sizeof(*ctrl)); + if (IS_ERR(ctrl)) + return PTR_ERR(ctrl); + + *size = ctrl->Size; + + nvkm_gsp_rm_ctrl_done(&gsp->internal.device.object, ctrl); + return 0; +} + +static int +r570_memsys_enable_raw_comp_mode(struct nvkm_gsp *gsp, bool enable) +{ + NV2080_CTRL_INTERNAL_MEMSYS_PROGRAM_RAW_COMPRESSION_MODE_PARAMS *ctrl; + int ret; + + ctrl = nvkm_gsp_rm_ctrl_get(&gsp->internal.device.subdevice, + NV2080_CTRL_CMD_INTERNAL_MEMSYS_PROGRAM_RAW_COMPRESSION_MODE, + sizeof(*ctrl)); + if (IS_ERR(ctrl)) + return PTR_ERR(ctrl); + + ctrl->bRawMode = enable; + + ret = nvkm_gsp_rm_ctrl_wr(&gsp->internal.device.subdevice, ctrl); + if (!ret) + nvkm_debug(&gsp->subdev, "memsys: Raw compression mode %s\n", + str_enabled_disabled(enable)); + + return ret; +} + +static bool +r570_need_raw_comp_war(struct nvkm_gsp *gsp, struct nvkm_device *device) +{ + return (device->card_type == GA100 || device->card_type == AD100) && + gsp->memsys.use_raw_mode_comptagline_alloc; +} + static void r570_fbsr_resume(struct nvkm_gsp *gsp) { @@ -33,6 +79,7 @@ r570_fbsr_resume(struct nvkm_gsp *gsp) struct nvkm_instmem *imem = device->imem; struct nvkm_instobj *iobj; struct nvkm_vmm *vmm; + int ret; /* Restore BAR2 page tables via BAR0 window, and re-enable BAR2. */ list_for_each_entry(iobj, &imem->boot, head) { @@ -54,6 +101,13 @@ r570_fbsr_resume(struct nvkm_gsp *gsp) vmm = nvkm_bar_bar1_vmm(device); vmm->func->flush(vmm, 0); + /* Re-enable raw mode if it was previously disabled */ + if (r570_need_raw_comp_war(gsp, device)) { + ret = r570_memsys_enable_raw_comp_mode(gsp, true); + if (ret) + nvkm_error(&gsp->subdev, "Failed to re-enable raw comp mode\n"); + } + /* Resume channel scheduling. */ r570_fbsr_suspend_channels(device->gsp, false); @@ -81,7 +135,7 @@ r570_fbsr_init(struct nvkm_gsp *gsp, struct sg_table *sgt, u64 size) ctrl->hClient = gsp->internal.client.object.handle; ctrl->hSysMem = memlist.handle; ctrl->sysmemAddrOfSuspendResumeData = gsp->sr.meta.addr; - ctrl->bEnteringGcoffState = 0; + ctrl->bEnteringGcoffState = 1; ret = nvkm_gsp_rm_ctrl_wr(&gsp->internal.device.subdevice, ctrl); if (ret) @@ -98,12 +152,29 @@ r570_fbsr_suspend(struct nvkm_gsp *gsp) struct nvkm_device *device = subdev->device; struct nvkm_instmem *imem = device->imem; struct nvkm_instobj *iobj; - u64 size; + u64 size, compbit_store_size; int ret; /* Stop channel scheduling. */ r570_fbsr_suspend_channels(gsp, true); + /* Temporarily disable raw mode to prevent FBSR restore operations from corrupting + * compressed surfaces. Required for ampere and ada. + * + * Nvidia bug #3172217 + */ + if (r570_need_raw_comp_war(gsp, device)) { + ret = r570_memsys_enable_raw_comp_mode(gsp, false); + if (ret) + return ret; + } + + ret = r570_fb_get_compbit_store_size(gsp, &compbit_store_size); + if (ret < 0) + return ret; + nvkm_debug(&gsp->subdev, "fbsr: Compbit backing store size: 0x%llx bytes\n", + compbit_store_size); + /* Save BAR2 allocations to system memory. */ list_for_each_entry(iobj, &imem->list, head) { if (iobj->preserve) { @@ -126,6 +197,8 @@ r570_fbsr_suspend(struct nvkm_gsp *gsp) size = gsp->fb.heap.size; size += gsp->fb.rsvd_size; size += gsp->fb.bios.vga_workspace.size; + size += compbit_store_size; + nvkm_debug(subdev, "fbsr: size: 0x%llx bytes\n", size); ret = nvkm_gsp_sg(device, size, &gsp->sr.fbsr); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/gsp.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/gsp.c index b45781cd0dfd..ea38a94211f4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/gsp.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/gsp.c @@ -10,6 +10,7 @@ #include "nvrm/gsp.h" #include "nvrm/rpcfn.h" #include "nvrm/msgfn.h" +#include "nvif/cl0080.h" #include #include @@ -137,6 +138,14 @@ r570_gsp_get_static_info(struct nvkm_gsp *gsp) } } + ret = r535_gsp_get_static_memsys_info(gsp); + if (ret) { + nvkm_error(&gsp->subdev, "Retrieving static memsys info failed\n"); + return ret; + } + nvkm_debug(&gsp->subdev, "memsys: Use raw mode for comptag allocations? %s\n", + str_yes_no(gsp->memsys.use_raw_mode_comptagline_alloc)); + return 0; } @@ -215,6 +224,32 @@ r570_gsp_set_rmargs(struct nvkm_gsp *gsp, bool resume) args->bDmemStack = 1; } +int +r570_gsp_gcx_ready(struct nvkm_gsp *gsp) +{ + NV2080_CTRL_INTERNAL_GCX_ENTRY_PREREQUISITE_PARAMS *ctrl; + int ret = 0; + + ctrl = nvkm_gsp_rm_ctrl_rd(&gsp->internal.device.subdevice, + NV2080_CTRL_CMD_INTERNAL_GCX_ENTRY_PREREQUISITE, + sizeof(*ctrl)); + if (IS_ERR(ctrl)) + return PTR_ERR(ctrl); + + if (ctrl->bIsGC6Satisfied) + ret |= NV_DEVICE_GC6_READY; + if (ctrl->bIsGCOFFSatisfied) + ret |= NV_DEVICE_GCOFF_READY; + + nvkm_debug(&gsp->subdev, + "GCX ready status: GC6=%s GCOFF=%s\n", + str_yes_no(ctrl->bIsGC6Satisfied), str_yes_no(ctrl->bIsGCOFFSatisfied)); + + nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl); + return ret; +} + + const struct nvkm_rm_api_gsp r570_gsp = { .set_rmargs = r570_gsp_set_rmargs, @@ -223,4 +258,5 @@ r570_gsp = { .xlat_mc_engine_idx = r570_gsp_xlat_mc_engine_idx, .drop_post_nocat_record = r570_gsp_drop_post_nocat_record, .sr_data_size = r570_gsp_sr_data_size, + .gcx_ready = r570_gsp_gcx_ready, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/fbsr.h b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/fbsr.h index 8af432375f7a..cb3e448415b4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/fbsr.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/fbsr.h @@ -16,4 +16,33 @@ typedef struct NV2080_CTRL_INTERNAL_FBSR_INIT_PARAMS { NV_DECLARE_ALIGNED(NvU64 sysmemAddrOfSuspendResumeData, 8); } NV2080_CTRL_INTERNAL_FBSR_INIT_PARAMS; +#define NV0080_CTRL_CMD_FB_GET_COMPBIT_STORE_INFO (0x801306) /* finn: Evaluated from "(FINN_NV01_DEVICE_0_FB_INTERFACE_ID << 8) | NV0080_CTRL_FB_GET_COMPBIT_STORE_INFO_PARAMS_MESSAGE_ID" */ + +typedef struct NV0080_CTRL_FB_GET_COMPBIT_STORE_INFO_PARAMS { + NV_DECLARE_ALIGNED(NvU64 Size, 8); + NV_DECLARE_ALIGNED(NvU64 Address, 8); + NvU32 AddressSpace; + NvU32 MaxCompbitLine; + NvU32 comptagsPerCacheLine; + NvU32 cacheLineSize; + NvU32 cacheLineSizePerSlice; + NvU32 cacheLineFetchAlignment; + NV_DECLARE_ALIGNED(NvU64 backingStoreBase, 8); + NvU32 gobsPerComptagPerSlice; + NvU32 backingStoreCbcBase; + NvU32 comptaglineAllocationPolicy; + NV_DECLARE_ALIGNED(NvU64 privRegionStartOffset, 8); + NvU32 cbcCoveragePerSlice; +} NV0080_CTRL_FB_GET_COMPBIT_STORE_INFO_PARAMS; + +#define NV0080_CTRL_CMD_FB_GET_COMPBIT_STORE_INFO_ADDRESS_SPACE_UNKNOWN 0 // ADDR_UNKNOWN +#define NV0080_CTRL_CMD_FB_GET_COMPBIT_STORE_INFO_ADDRESS_SPACE_SYSMEM 1 // ADDR_SYSMEM +#define NV0080_CTRL_CMD_FB_GET_COMPBIT_STORE_INFO_ADDRESS_SPACE_FBMEM 2 // ADDR_FBMEM + +#define NV2080_CTRL_CMD_INTERNAL_MEMSYS_PROGRAM_RAW_COMPRESSION_MODE (0x20800a6f) /* finn: Evaluated from "(FINN_NV20_SUBDEVICE_0_INTERNAL_INTERFACE_ID << 8) | NV2080_CTRL_INTERNAL_MEMSYS_PROGRAM_RAW_COMPRESSION_MODE_PARAMS_MESSAGE_ID" */ + +typedef struct NV2080_CTRL_INTERNAL_MEMSYS_PROGRAM_RAW_COMPRESSION_MODE_PARAMS { + NvBool bRawMode; +} NV2080_CTRL_INTERNAL_MEMSYS_PROGRAM_RAW_COMPRESSION_MODE_PARAMS; + #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/gsp.h b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/gsp.h index c458569af9d7..2814629fddd2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/gsp.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/gsp.h @@ -639,4 +639,11 @@ typedef struct GSP_FMC_BOOT_PARAMS } GSP_FMC_BOOT_PARAMS; #define GSP_FW_HEAP_PARAM_BASE_RM_SIZE_GH100 (14 << 20) // Hopper+ + +#define NV2080_CTRL_CMD_INTERNAL_GCX_ENTRY_PREREQUISITE (0x2080a7d7) + +typedef struct NV2080_CTRL_INTERNAL_GCX_ENTRY_PREREQUISITE_PARAMS { + NvBool bIsGC6Satisfied; + NvBool bIsGCOFFSatisfied; +} NV2080_CTRL_INTERNAL_GCX_ENTRY_PREREQUISITE_PARAMS; #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/rm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/rm.h index e9ac47d86b69..17480d4e527a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/rm.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/rm.h @@ -40,6 +40,7 @@ struct nvkm_rm_api { void (*drop_send_user_shared_data)(struct nvkm_gsp *); void (*drop_post_nocat_record)(struct nvkm_gsp *); u32 (*sr_data_size)(struct nvkm_gsp *); + int (*gcx_ready)(struct nvkm_gsp *gsp); } *gsp; const struct nvkm_rm_api_rpc { @@ -174,6 +175,8 @@ int r535_gr_chan_new(struct nvkm_gr *, struct nvkm_chan *, const struct nvkm_ocl int r535_gr_promote_ctx(struct r535_gr *, bool golden, struct nvkm_vmm *, struct nvkm_memory **pctxbuf_mem, struct nvkm_vma **pctxbuf_vma, struct nvkm_gsp_object *chan); +int r570_gsp_gcx_ready(struct nvkm_gsp *gsp); +int r535_gsp_get_static_memsys_info(struct nvkm_gsp *gsp); extern const struct nvkm_rm_api_engine r535_nvdec; extern const struct nvkm_rm_api_engine r535_nvenc; extern const struct nvkm_rm_api_engine r535_nvjpg; diff --git a/drivers/gpu/drm/radeon/radeon_bios.c b/drivers/gpu/drm/radeon/radeon_bios.c index 8595f4c6e2e5..1da6bf1c17ff 100644 --- a/drivers/gpu/drm/radeon/radeon_bios.c +++ b/drivers/gpu/drm/radeon/radeon_bios.c @@ -49,6 +49,7 @@ static bool igp_read_bios_from_vram(struct radeon_device *rdev) { uint8_t __iomem *bios; + uint8_t val1, val2; resource_size_t vram_base; resource_size_t size = 256 * 1024; /* ??? */ @@ -63,7 +64,10 @@ static bool igp_read_bios_from_vram(struct radeon_device *rdev) return false; } - if (size == 0 || bios[0] != 0x55 || bios[1] != 0xaa) { + val1 = readb(&bios[0]); + val2 = readb(&bios[1]); + + if (size == 0 || val1 != 0x55 || val2 != 0xaa) { iounmap(bios); return false; } diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.h b/drivers/gpu/drm/virtio/virtgpu_drv.h index 9df4c7117341..f3bbbe4468bf 100644 --- a/drivers/gpu/drm/virtio/virtgpu_drv.h +++ b/drivers/gpu/drm/virtio/virtgpu_drv.h @@ -114,6 +114,8 @@ struct virtio_gpu_object { bool dumb; bool created; bool attached; + /* a guest-bound transfer is queued and its mapping not yet synced */ + bool from_host_pending; bool host3d_blob, guest_blob; uint32_t blob_mem, blob_flags; @@ -196,6 +198,9 @@ struct virtio_gpu_vbuffer { struct list_head list; uint32_t seqno; + + /* guest-bound transfer whose shmem backing needs a CPU sync */ + bool sync_for_cpu; }; struct virtio_gpu_output { diff --git a/drivers/gpu/drm/virtio/virtgpu_gem.c b/drivers/gpu/drm/virtio/virtgpu_gem.c index 66c3f6f74e9c..d2f0b8a3f172 100644 --- a/drivers/gpu/drm/virtio/virtgpu_gem.c +++ b/drivers/gpu/drm/virtio/virtgpu_gem.c @@ -45,7 +45,7 @@ static int virtio_gpu_gem_create(struct drm_file *file, ret = drm_gem_handle_create(file, &obj->base.base, &handle); if (ret) { - drm_gem_object_release(&obj->base.base); + drm_gem_object_put(&obj->base.base); return ret; } diff --git a/drivers/gpu/drm/virtio/virtgpu_ioctl.c b/drivers/gpu/drm/virtio/virtgpu_ioctl.c index 3d8e4ccdb7c1..81e70a12b356 100644 --- a/drivers/gpu/drm/virtio/virtgpu_ioctl.c +++ b/drivers/gpu/drm/virtio/virtgpu_ioctl.c @@ -185,7 +185,7 @@ static int virtio_gpu_resource_create_ioctl(struct drm_device *dev, void *data, ret = drm_gem_handle_create(file, obj, &handle); if (ret) { - drm_gem_object_release(obj); + drm_gem_object_put(obj); return ret; } @@ -261,6 +261,27 @@ static int virtio_gpu_transfer_from_host_ioctl(struct drm_device *dev, if (ret != 0) goto err_put_free; + if (virtio_gpu_is_shmem(bo) && virtio_gpu_use_dma_api(vgdev->vdev)) { + /* + * The sync on completion restores the whole mapping, so an + * earlier transfer has to be done before this one snapshots it. + * Otherwise the snapshot predates anything the CPU wrote once + * that transfer's fence signalled, and the later sync would + * discard it. Nothing can add a fence behind our back here, + * since doing so takes the reservation we already hold. + * This writes the pages, so it waits as a writer does. READ + * usage covers existing readers. + */ + long wait = dma_resv_wait_timeout(objs->objs[0]->resv, + DMA_RESV_USAGE_READ, true, + MAX_SCHEDULE_TIMEOUT); + + if (wait < 0) { + ret = wait; + goto err_unlock; + } + } + fence = virtio_gpu_fence_alloc(vgdev, vgdev->fence_drv.context, 0); if (!fence) { ret = -ENOMEM; @@ -320,6 +341,28 @@ static int virtio_gpu_transfer_to_host_ioctl(struct drm_device *dev, void *data, if (ret != 0) goto err_put_free; + /* + * A transfer the other way may have queued without yet syncing + * its mapping. Pushing the guest pages into it now would + * discard what the device wrote there, so wait for that sync: + * it runs before the fence it belongs to is signalled. The + * flag is only set under this reservation, so it cannot appear + * behind our back, and the acquire pairs with the release in + * that sync, so finding it clear means the pages it wrote are + * visible here too. + */ + if (smp_load_acquire(&bo->from_host_pending)) { + long wait = dma_resv_wait_timeout(objs->objs[0]->resv, + DMA_RESV_USAGE_WRITE, + true, + MAX_SCHEDULE_TIMEOUT); + + if (wait < 0) { + ret = wait; + goto err_unlock; + } + } + ret = -ENOMEM; fence = virtio_gpu_fence_alloc(vgdev, vgdev->fence_drv.context, 0); @@ -557,14 +600,14 @@ static int virtio_gpu_resource_create_blob_ioctl(struct drm_device *dev, if (params.blob_flags & VIRTGPU_BLOB_FLAG_USE_CROSS_DEVICE) { ret = virtio_gpu_resource_assign_uuid(vgdev, bo); if (ret) { - drm_gem_object_release(obj); + drm_gem_object_put(obj); return ret; } } ret = drm_gem_handle_create(file, obj, &handle); if (ret) { - drm_gem_object_release(obj); + drm_gem_object_put(obj); return ret; } diff --git a/drivers/gpu/drm/virtio/virtgpu_plane.c b/drivers/gpu/drm/virtio/virtgpu_plane.c index 62955acd1f8e..10718bc73693 100644 --- a/drivers/gpu/drm/virtio/virtgpu_plane.c +++ b/drivers/gpu/drm/virtio/virtgpu_plane.c @@ -595,6 +595,7 @@ struct drm_plane *virtio_gpu_plane_init(struct virtio_gpu_device *vgdev, struct drm_plane *plane; const uint32_t *formats; int nformats; + int ret; if (type == DRM_PLANE_TYPE_CURSOR) { formats = virtio_gpu_cursor_formats; @@ -620,5 +621,17 @@ struct drm_plane *virtio_gpu_plane_init(struct virtio_gpu_device *vgdev, drm_plane_create_blend_mode_property(plane, BIT(DRM_MODE_BLEND_PREMULTI)); + if (type == DRM_PLANE_TYPE_CURSOR) { + /* + * The cursor plane exposes a format with an alpha channel, + * which requires a blend mode property. The host blends + * premultiplied alpha, matching the property's default. + */ + ret = drm_plane_create_blend_mode_property(plane, + BIT(DRM_MODE_BLEND_PREMULTI)); + if (ret) + return ERR_PTR(ret); + } + return plane; } diff --git a/drivers/gpu/drm/virtio/virtgpu_prime.c b/drivers/gpu/drm/virtio/virtgpu_prime.c index 149e6bcb5878..ebf471044d06 100644 --- a/drivers/gpu/drm/virtio/virtgpu_prime.c +++ b/drivers/gpu/drm/virtio/virtgpu_prime.c @@ -349,7 +349,7 @@ struct drm_gem_object *virtgpu_gem_prime_import(struct drm_device *dev, } } - if (!vgdev->has_resource_blob) + if (!vgdev->has_resource_blob || vgdev->has_virgl_3d) return drm_gem_prime_import(dev, buf); bo = kzalloc_obj(*bo); diff --git a/drivers/gpu/drm/virtio/virtgpu_submit.c b/drivers/gpu/drm/virtio/virtgpu_submit.c index 32cb1e4aa425..3d35326dd904 100644 --- a/drivers/gpu/drm/virtio/virtgpu_submit.c +++ b/drivers/gpu/drm/virtio/virtgpu_submit.c @@ -389,10 +389,13 @@ static int virtio_gpu_init_submit(struct virtio_gpu_submit *submit, if ((exbuf->flags & VIRTGPU_EXECBUF_FENCE_FD_OUT) || exbuf->num_out_syncobjs || exbuf->num_bo_handles || - drm_fence_event) + drm_fence_event) { out_fence = virtio_gpu_fence_alloc(vgdev, fence_ctx, ring_idx); - else + if (!out_fence) + return -ENOMEM; + } else { out_fence = NULL; + } if (drm_fence_event) { err = virtio_gpu_fence_event_create(dev, file, out_fence, ring_idx); @@ -538,6 +541,10 @@ int virtio_gpu_execbuffer_ioctl(struct drm_device *dev, void *data, virtio_gpu_process_post_deps(&submit); virtio_gpu_complete_submit(&submit); cleanup: + if (ret && submit.out_fence && submit.out_fence->e) { + drm_event_cancel_free(dev, &submit.out_fence->e->base); + submit.out_fence->e = NULL; + } virtio_gpu_cleanup_submit(&submit); return ret; diff --git a/drivers/gpu/drm/virtio/virtgpu_vq.c b/drivers/gpu/drm/virtio/virtgpu_vq.c index c02c03c10d92..d99fb9e326e8 100644 --- a/drivers/gpu/drm/virtio/virtgpu_vq.c +++ b/drivers/gpu/drm/virtio/virtgpu_vq.c @@ -256,6 +256,33 @@ void virtio_gpu_dequeue_ctrl_func(struct work_struct *work) } while (!virtqueue_enable_cb(vgdev->ctrlq.vq)); spin_unlock(&vgdev->ctrlq.qlock); + /* + * Sync guest-bound transfers before signalling anything, so that a + * waiter cannot read the backing pages while what the device wrote is + * still in a bounce buffer. This cannot be folded into the loop below: + * virtio_gpu_fence_event_process() also signals every earlier fence in + * the same context, so any entry there may signal this entry's fence. + */ + list_for_each_entry(entry, &reclaim_list, list) { + if (entry->sync_for_cpu) { + struct virtio_gpu_object *bo = + gem_to_virtio_gpu_obj(entry->objs->objs[0]); + + dma_sync_sgtable_for_cpu(vgdev->vdev->dev.parent, + bo->base.sgt, DMA_FROM_DEVICE); + /* + * Release, so a transfer the other way that skips its + * wait on the strength of this cannot go on to read + * the backing pages before the sync above is visible. + * Nothing orders the two otherwise: where the mapping + * bounces on a coherent device the sync is a plain + * copy, and dma_direct_sync_sg_for_cpu() emits its + * barrier only for the non-coherent case. + */ + smp_store_release(&bo->from_host_pending, false); + } + } + list_for_each_entry(entry, &reclaim_list, list) { resp = (struct virtio_gpu_ctrl_hdr *)entry->resp_buf; @@ -1278,12 +1305,31 @@ void virtio_gpu_cmd_transfer_from_host_3d(struct virtio_gpu_device *vgdev, struct virtio_gpu_object *bo = gem_to_virtio_gpu_obj(objs->objs[0]); struct virtio_gpu_transfer_host_3d *cmd_p; struct virtio_gpu_vbuffer *vbuf; + bool use_dma_api = virtio_gpu_use_dma_api(vgdev->vdev); cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p)); memset(cmd_p, 0, sizeof(*cmd_p)); vbuf->objs = objs; + if (virtio_gpu_is_shmem(bo) && use_dma_api) { + /* + * The device writes only the requested box, so prime the + * mapping with the current contents: otherwise the sync on + * completion would hand back whatever a bounce buffer held for + * the regions the device does not touch. + */ + dma_sync_sgtable_for_device(vgdev->vdev->dev.parent, + bo->base.sgt, DMA_TO_DEVICE); + vbuf->sync_for_cpu = true; + /* + * Set under the reservation the caller holds, so a transfer + * the other way cannot miss it and push the guest pages into + * the mapping while the device still owns it. + */ + WRITE_ONCE(bo->from_host_pending, true); + } + cmd_p->hdr.type = cpu_to_le32(VIRTIO_GPU_CMD_TRANSFER_FROM_HOST_3D); cmd_p->hdr.ctx_id = cpu_to_le32(ctx_id); cmd_p->resource_id = cpu_to_le32(bo->hw_res_handle); diff --git a/drivers/gpu/drm/virtio/virtgpu_vram.c b/drivers/gpu/drm/virtio/virtgpu_vram.c index 5b4a3ab81cd5..01241ce4d07c 100644 --- a/drivers/gpu/drm/virtio/virtgpu_vram.c +++ b/drivers/gpu/drm/virtio/virtgpu_vram.c @@ -215,16 +215,12 @@ int virtio_gpu_vram_create(struct virtio_gpu_device *vgdev, /* Create fake offset */ ret = drm_gem_create_mmap_offset(obj); - if (ret) { - kfree(vram); - return ret; - } + if (ret) + goto err_release_obj; ret = virtio_gpu_resource_id_get(vgdev, &vram->base.hw_res_handle); - if (ret) { - kfree(vram); - return ret; - } + if (ret) + goto err_release_obj; virtio_gpu_cmd_resource_create_blob(vgdev, &vram->base, params, NULL, 0); @@ -240,6 +236,11 @@ int virtio_gpu_vram_create(struct virtio_gpu_device *vgdev, *bo_ptr = &vram->base; return 0; + +err_release_obj: + drm_gem_object_release(obj); + kfree(vram); + return ret; } void virtio_gpu_vram_map_deferred(struct virtio_gpu_object_vram *vram) diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig index a81bf51cbcf1..c43e824428f0 100644 --- a/drivers/hid/Kconfig +++ b/drivers/hid/Kconfig @@ -1210,7 +1210,8 @@ config HID_STEELSERIES depends on USB_HID help Support for Steelseries SRW-S1 steering wheel, and the Steelseries - Arctis 1 Wireless for XBox headset. + Arctis 1 Wireless for XBox, Arctis 7 (2018), Arctis 9, and Arctis + Nova headsets. config HID_SUNPLUS tristate "Sunplus wireless desktop" diff --git a/drivers/hid/amd-sfh-hid/amd_sfh_common.h b/drivers/hid/amd-sfh-hid/amd_sfh_common.h index f8c6b7fc34fb..3d29119a3765 100644 --- a/drivers/hid/amd-sfh-hid/amd_sfh_common.h +++ b/drivers/hid/amd-sfh-hid/amd_sfh_common.h @@ -13,11 +13,15 @@ #include #include #include +#include #include "amd_sfh_hid.h" #define PCI_DEVICE_ID_AMD_MP2 0x15E4 #define PCI_DEVICE_ID_AMD_MP2_1_1 0x164A +/* The BAR 2 size must cover the highest register offset (0x10958) */ +#define AMD_SFH_MIN_BAR_SIZE SZ_128K + #define AMD_C2P_MSG(regno) (0x10500 + ((regno) * 4)) #define AMD_P2C_MSG(regno) (0x10680 + ((regno) * 4)) diff --git a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c index eda26a094d3f..061a63519d44 100644 --- a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c +++ b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c @@ -497,6 +497,16 @@ static int amd_mp2_pci_probe(struct pci_dev *pdev, const struct pci_device_id *i if (rc) return rc; + if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) { + dev_err(&pdev->dev, "BAR 2 is not IORESOURCE_MEM\n"); + return -ENODEV; + } + + if (pci_resource_len(pdev, 2) < AMD_SFH_MIN_BAR_SIZE) { + dev_err(&pdev->dev, "BAR 2 is too small\n"); + return -EINVAL; + } + rc = pcim_iomap_regions(pdev, BIT(2), DRIVER_NAME); if (rc) return rc; diff --git a/drivers/hid/bpf/hid_bpf_dispatch.c b/drivers/hid/bpf/hid_bpf_dispatch.c index 44671dbdeca8..6b467c74601c 100644 --- a/drivers/hid/bpf/hid_bpf_dispatch.c +++ b/drivers/hid/bpf/hid_bpf_dispatch.c @@ -359,7 +359,7 @@ hid_bpf_release_context(struct hid_bpf_ctx *ctx) static int __hid_bpf_hw_check_params(struct hid_bpf_ctx *ctx, __u8 *buf, size_t *buf__sz, - enum hid_report_type rtype) + enum hid_report_type rtype, bool hw_request) { struct hid_report_enum *report_enum; struct hid_report *report; @@ -388,6 +388,10 @@ __hid_bpf_hw_check_params(struct hid_bpf_ctx *ctx, __u8 *buf, size_t *buf__sz, report_len = hid_report_len(report); + /* unnumbered reports need to have a report ID reserved in the first byte */ + if (hw_request && report_enum->numbered == 0) + report_len += 1; + if (*buf__sz > report_len) *buf__sz = report_len; @@ -420,7 +424,7 @@ hid_bpf_hw_request(struct hid_bpf_ctx *ctx, __u8 *buf, size_t buf__sz, return -EDEADLOCK; /* check arguments */ - ret = __hid_bpf_hw_check_params(ctx, buf, &size, rtype); + ret = __hid_bpf_hw_check_params(ctx, buf, &size, rtype, true); if (ret) return ret; @@ -449,7 +453,7 @@ hid_bpf_hw_request(struct hid_bpf_ctx *ctx, __u8 *buf, size_t buf__sz, (u64)(long)ctx, true); /* prevent infinite recursions */ - if (ret > size) + if (ret > (ssize_t)size) ret = size; if (ret > 0) memcpy(buf, dma_data, ret); @@ -480,7 +484,7 @@ hid_bpf_hw_output_report(struct hid_bpf_ctx *ctx, __u8 *buf, size_t buf__sz) return -EDEADLOCK; /* check arguments */ - ret = __hid_bpf_hw_check_params(ctx, buf, &size, HID_OUTPUT_REPORT); + ret = __hid_bpf_hw_check_params(ctx, buf, &size, HID_OUTPUT_REPORT, true); if (ret) return ret; @@ -506,7 +510,7 @@ __hid_bpf_input_report(struct hid_bpf_ctx *ctx, enum hid_report_type type, u8 *b return -EDEADLOCK; /* check arguments */ - ret = __hid_bpf_hw_check_params(ctx, buf, &size, type); + ret = __hid_bpf_hw_check_params(ctx, buf, &size, type, false); if (ret) return ret; diff --git a/drivers/hid/hid-alps.c b/drivers/hid/hid-alps.c index 67179e3fe39b..0556cb5645eb 100644 --- a/drivers/hid/hid-alps.c +++ b/drivers/hid/hid-alps.c @@ -407,6 +407,8 @@ static int u1_raw_event(struct alps_dev *hdata, u8 *data, int size) return 1; case U1_SP_ABSOLUTE_REPORT_ID: + if (!hdata->input2) + return 0; sp_x = get_unaligned_le16(data+2); sp_y = get_unaligned_le16(data+4); @@ -738,7 +740,6 @@ static int alps_input_configured(struct hid_device *hdev, struct hid_input *hi) goto exit; } - data->input2 = input2; input2->phys = input->phys; input2->name = "DualPoint Stick"; input2->id.bustype = BUS_I2C; @@ -762,11 +763,12 @@ static int alps_input_configured(struct hid_device *hdev, struct hid_input *hi) __set_bit(INPUT_PROP_POINTER, input2->propbit); __set_bit(INPUT_PROP_POINTING_STICK, input2->propbit); - if (input_register_device(data->input2)) { + if (input_register_device(input2)) { input_free_device(input2); ret = -ENOENT; goto exit; } + data->input2 = input2; } exit: @@ -823,6 +825,24 @@ static int alps_probe(struct hid_device *hdev, const struct hid_device_id *id) return 0; } +static void alps_remove(struct hid_device *hdev) +{ + struct alps_dev *data = hid_get_drvdata(hdev); + + /* + * input2 ("DualPoint Stick") is allocated separately and is not + * tracked in hdev->inputs, so the default remove path + * (hid_hw_stop -> hidinput_disconnect) does not unregister it. + * + * Stop the device first so that no URB callback can touch input2 + * while it is being unregistered, then drop it explicitly. + */ + hid_hw_stop(hdev); + + if (data->input2) + input_unregister_device(data->input2); +} + static const struct hid_device_id alps_id[] = { { HID_DEVICE(HID_BUS_ANY, HID_GROUP_ANY, USB_VENDOR_ID_ALPS_JP, HID_DEVICE_ID_ALPS_U1_DUAL) }, @@ -845,6 +865,7 @@ static struct hid_driver alps_driver = { .input_configured = alps_input_configured, .resume = pm_ptr(alps_post_resume), .reset_resume = pm_ptr(alps_post_reset), + .remove = alps_remove, }; module_hid_driver(alps_driver); diff --git a/drivers/hid/hid-corsair-void.c b/drivers/hid/hid-corsair-void.c index 071a663a6c26..7be8c974d11e 100644 --- a/drivers/hid/hid-corsair-void.c +++ b/drivers/hid/hid-corsair-void.c @@ -51,20 +51,23 @@ /* Receiver report information: (ID 102) */ /* -------------------------------------------------------------------------- */ /* - * When queried, the recevier responds with 4 bytes to describe the firmware - * The first 2 bytes are for the receiver, the second 2 are the headset + * When queried, the receiver responds with 5 bytes to describe the firmware + * The first byte is the ID, then 2 bytes for the receiver, then 2 for the headset * The headset firmware version will be 0 if no headset is connected * - * 0: Recevier firmware major version + * 0: Report ID + * 102 for the firmware packet + * + * 1: Receiver firmware major version * Major version of the receiver's firmware * - * 1: Recevier firmware minor version + * 2: Receiver firmware minor version * Minor version of the receiver's firmware * - * 2: Headset firmware major version + * 3: Headset firmware major version * Major version of the headset's firmware * - * 3: Headset firmware minor version + * 4: Headset firmware minor version * Minor version of the headset's firmware */ /* -------------------------------------------------------------------------- */ diff --git a/drivers/hid/hid-elecom.c b/drivers/hid/hid-elecom.c index 37d88ce57f67..d21ead40bb07 100644 --- a/drivers/hid/hid-elecom.c +++ b/drivers/hid/hid-elecom.c @@ -79,7 +79,7 @@ static const __u8 *elecom_report_fixup(struct hid_device *hdev, __u8 *rdesc, case USB_DEVICE_ID_ELECOM_M_XT3URBK_00FB: case USB_DEVICE_ID_ELECOM_M_XT3URBK_018F: case USB_DEVICE_ID_ELECOM_M_XT3DRBK_00FC: - case USB_DEVICE_ID_ELECOM_M_XT4DRBK: + case USB_DEVICE_ID_ELECOM_M_XT4DRBK_00FD: /* * Report descriptor format: * 12: button bit count @@ -104,6 +104,7 @@ static const __u8 *elecom_report_fixup(struct hid_device *hdev, __u8 *rdesc, mouse_button_fixup(hdev, rdesc, *rsize, 12, 30, 14, 20, 8); break; case USB_DEVICE_ID_ELECOM_M_XT3DRBK_018C: + case USB_DEVICE_ID_ELECOM_M_XT4DRBK_018E: /* * Report descriptor format: * 22: button bit count @@ -148,7 +149,8 @@ static const struct hid_device_id elecom_devices[] = { { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3URBK_018F) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3DRBK_00FC) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3DRBK_018C) }, - { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT4DRBK) }, + { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT4DRBK_00FD) }, + { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT4DRBK_018E) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_DT1URBK) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_DT1DRBK) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_DT2DRBK) }, diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h index b3aca5aa9176..cb7f16d75fce 100644 --- a/drivers/hid/hid-ids.h +++ b/drivers/hid/hid-ids.h @@ -477,7 +477,8 @@ #define USB_DEVICE_ID_ELECOM_M_XT3URBK_018F 0x018f #define USB_DEVICE_ID_ELECOM_M_XT3DRBK_00FC 0x00fc #define USB_DEVICE_ID_ELECOM_M_XT3DRBK_018C 0x018c -#define USB_DEVICE_ID_ELECOM_M_XT4DRBK 0x00fd +#define USB_DEVICE_ID_ELECOM_M_XT4DRBK_00FD 0x00fd +#define USB_DEVICE_ID_ELECOM_M_XT4DRBK_018E 0x018e #define USB_DEVICE_ID_ELECOM_M_DT1URBK 0x00fe #define USB_DEVICE_ID_ELECOM_M_DT1DRBK 0x00ff #define USB_DEVICE_ID_ELECOM_M_DT2DRBK 0x018d @@ -765,6 +766,7 @@ #define I2C_DEVICE_ID_ITE_LENOVO_LEGION_Y720 0x837a #define USB_DEVICE_ID_ITE_LENOVO_YOGA900 0x8396 #define I2C_DEVICE_ID_ITE_LENOVO_YOGA_SLIM_7X_KEYBOARD 0x8987 +#define I2C_DEVICE_ID_ITE_LENOVO_YOGA_SLIM_7X_G11_KEYBOARD 0x83db #define USB_DEVICE_ID_ITE8595 0x8595 #define USB_DEVICE_ID_ITE_MEDION_E1239T 0xce50 @@ -1296,6 +1298,9 @@ #define USB_DEVICE_ID_SAMSUNG_WIRELESS_UNIVERSAL_KBD 0xa006 #define USB_DEVICE_ID_SAMSUNG_WIRELESS_MULTI_HOGP_KBD 0xa064 +#define USB_VENDOR_ID_SDINNOVATION 0x36ae +#define USB_DEVICE_ID_SDINNOVATION_GAMING_KBD 0xfeab + #define USB_VENDOR_ID_SEMICO 0x1a2c #define USB_DEVICE_ID_SEMICO_USB_KEYKOARD 0x0023 #define USB_DEVICE_ID_SEMICO_USB_KEYKOARD2 0x0027 @@ -1395,6 +1400,7 @@ #define USB_VENDOR_ID_STEELSERIES 0x1038 #define USB_DEVICE_ID_STEELSERIES_SRWS1 0x1410 #define USB_DEVICE_ID_STEELSERIES_ARCTIS_1_X 0x12b6 +#define USB_DEVICE_ID_STEELSERIES_ARCTIS_7_2018 0x12ad #define USB_DEVICE_ID_STEELSERIES_ARCTIS_9 0x12c2 #define USB_DEVICE_ID_STEELSERIES_ARCTIS_NOVA_5_X 0x2253 #define USB_DEVICE_ID_STEELSERIES_ARCTIS_NOVA_7 0x2202 @@ -1504,6 +1510,9 @@ #define USB_DEVICE_ID_ASUS_MD_5110 0x5110 #define USB_DEVICE_ID_TURBOX_TOUCHSCREEN_MOSART 0x7100 +#define USB_VENDOR_ID_TURTLE_BEACH 0x10f5 +#define USB_DEVICE_ID_TB_VELOCITYONE_RACE 0x7077 + #define USB_VENDOR_ID_TWINHAN 0x6253 #define USB_DEVICE_ID_TWINHAN_IR_REMOTE 0x0100 diff --git a/drivers/hid/hid-logitech-dj.c b/drivers/hid/hid-logitech-dj.c index 1d619d2345e1..390d106d5dc1 100644 --- a/drivers/hid/hid-logitech-dj.c +++ b/drivers/hid/hid-logitech-dj.c @@ -121,7 +121,6 @@ enum recvr_type { recvr_type_27mhz, recvr_type_bluetooth, recvr_type_dinovo, - recvr_type_bolt, }; struct dj_report { @@ -1157,10 +1156,6 @@ static void logi_hidpp_recv_queue_notif(struct hid_device *hdev, logi_hidpp_dev_conn_notif_equad(hdev, hidpp_report, &workitem); workitem.reports_supported |= STD_KEYBOARD; break; - case 0x10: - device_type = "Bolt"; - logi_hidpp_dev_conn_notif_equad(hdev, hidpp_report, &workitem); - break; } /* custom receiver device (eg. powerplay) */ @@ -1750,24 +1745,6 @@ static int logi_dj_hidpp_event(struct hid_device *hdev, dj_dev = djrcv_dev->paired_dj_devices[device_index]; - /* - * Bolt receivers send explicit unpair notifications as HID++ events; - * queue device removal when we receive one. - */ - if (djrcv_dev->type == recvr_type_bolt && - hidpp_report->report_id == REPORT_ID_HIDPP_SHORT && - hidpp_report->sub_id == REPORT_TYPE_NOTIF_DEVICE_UNPAIRED) { - struct dj_workitem workitem = { - .device_index = device_index, - .type = WORKITEM_TYPE_UNPAIRED, - }; - - kfifo_in(&djrcv_dev->notif_fifo, &workitem, sizeof(workitem)); - schedule_work(&djrcv_dev->work); - spin_unlock_irqrestore(&djrcv_dev->lock, flags); - return false; - } - /* * With 27 MHz receivers, we do not get an explicit unpair event, * remove the old device if the user has paired a *different* device. @@ -1907,9 +1884,6 @@ static int logi_dj_probe(struct hid_device *hdev, * treat these as logitech-dj interfaces then this causes input events * reported through this extra interface to not be reported correctly. * To avoid this, we treat these as generic-hid devices. - * - * Bolt receivers only use LOGITECH_DJ_INTERFACE_NUMBER for receiver - * reporting. Treat all other Bolt interfaces as generic-hid devices. */ switch (id->driver_data) { case recvr_type_dj: no_dj_interfaces = 3; break; @@ -1923,20 +1897,10 @@ static int logi_dj_probe(struct hid_device *hdev, } if (hid_is_usb(hdev)) { intf = to_usb_interface(hdev->dev.parent); - if (intf) { - bool generic_hid_interface; - - if (id->driver_data == recvr_type_bolt) - generic_hid_interface = - intf->altsetting->desc.bInterfaceNumber != - LOGITECH_DJ_INTERFACE_NUMBER; - else - generic_hid_interface = - intf->altsetting->desc.bInterfaceNumber >= no_dj_interfaces; - if (generic_hid_interface) { - hdev->quirks |= HID_QUIRK_INPUT_PER_APP; - return hid_hw_start(hdev, HID_CONNECT_DEFAULT); - } + if (intf && intf->altsetting->desc.bInterfaceNumber >= + no_dj_interfaces) { + hdev->quirks |= HID_QUIRK_INPUT_PER_APP; + return hid_hw_start(hdev, HID_CONNECT_DEFAULT); } } @@ -2139,10 +2103,6 @@ static const struct hid_device_id logi_dj_receivers[] = { HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_NANO_RECEIVER_LIGHTSPEED_1_3), .driver_data = recvr_type_gaming_hidpp_ls_1_3}, - { /* Logitech Bolt receiver (0xc548) */ - HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, - USB_DEVICE_ID_LOGITECH_BOLT_RECEIVER), - .driver_data = recvr_type_bolt}, { /* Logitech lightspeed receiver (0xc54d) */ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_NANO_RECEIVER_LIGHTSPEED_1_4), diff --git a/drivers/hid/hid-logitech-hidpp.c b/drivers/hid/hid-logitech-hidpp.c index 1504de32b1c8..9646380febac 100644 --- a/drivers/hid/hid-logitech-hidpp.c +++ b/drivers/hid/hid-logitech-hidpp.c @@ -988,8 +988,7 @@ static int hidpp_root_get_protocol_version(struct hidpp_device *hidpp) } /* the device might not be connected */ - if (ret == HIDPP_ERROR_CONNECT_FAIL || - ret == HIDPP_ERROR_RESOURCE_ERROR || + if (ret == HIDPP_ERROR_RESOURCE_ERROR || ret == HIDPP_ERROR_UNKNOWN_DEVICE) return -EIO; @@ -4405,50 +4404,8 @@ static int hidpp_initialize_battery(struct hidpp_device *hidpp) return ret; } -static bool hidpp_is_bolt_child(struct hid_device *hdev) -{ - struct device *parent = hdev->dev.parent; - struct hid_device *receiver_hdev; - - if (!parent) - return false; - - receiver_hdev = to_hid_device(parent); - return receiver_hdev->vendor == USB_VENDOR_ID_LOGITECH && - receiver_hdev->product == USB_DEVICE_ID_LOGITECH_BOLT_RECEIVER; -} - -static int hidpp_bolt_init(struct hidpp_device *hidpp) -{ - struct hid_device *hdev = hidpp->hid_dev; - char *name; - int ret; - - ret = hidpp_serial_init(hidpp); - if (ret) - return ret; - - name = hidpp_get_device_name(hidpp); - if (!name) - return -EIO; - - snprintf(hdev->name, sizeof(hdev->name), "%s", name); - dbg_hid("HID++ Bolt: Got name: %s\n", name); - - kfree(name); - return 0; -} - -static int hidpp_receiver_init(struct hidpp_device *hidpp) -{ - if (hidpp_is_bolt_child(hidpp->hid_dev)) - return hidpp_bolt_init(hidpp); - - return hidpp_unifying_init(hidpp); -} - /* Get name + serial for USB and Bluetooth HID++ devices */ -static void hidpp_non_receiver_init(struct hidpp_device *hidpp) +static void hidpp_non_unifying_init(struct hidpp_device *hidpp) { struct hid_device *hdev = hidpp->hid_dev; char *name; @@ -4800,9 +4757,9 @@ static int hidpp_probe(struct hid_device *hdev, const struct hid_device_id *id) /* Get name + serial, store in hdev->name + hdev->uniq */ if (id->group == HID_GROUP_LOGITECH_DJ_DEVICE) - hidpp_receiver_init(hidpp); + hidpp_unifying_init(hidpp); else - hidpp_non_receiver_init(hidpp); + hidpp_non_unifying_init(hidpp); if (hidpp->quirks & HIDPP_QUIRK_DELAYED_INIT) connect_mask &= ~HID_CONNECT_HIDINPUT; @@ -4924,6 +4881,8 @@ static const struct hid_device_id hidpp_devices[] = { HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, 0xC08D) }, { /* Logitech G502 X Plus Wireless Gaming Mouse over USB */ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, 0xC095) }, + { /* Logitech G502 X Lightspeed Wireless Gaming Mouse over USB */ + HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, 0xC098) }, { /* Logitech G703 Gaming Mouse over USB */ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, 0xC087) }, { /* Logitech G703 Hero Gaming Mouse over USB */ diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c index 451c7324e6a0..4e19a0c4d238 100644 --- a/drivers/hid/hid-multitouch.c +++ b/drivers/hid/hid-multitouch.c @@ -79,6 +79,7 @@ MODULE_LICENSE("GPL"); #define MT_QUIRK_APPLE_TOUCHBAR BIT(23) #define MT_QUIRK_YOGABOOK9I BIT(24) #define MT_QUIRK_KEEP_LATENCY_ON_CLOSE BIT(25) +#define MT_QUIRK_IGNORE_FEATURE_ID_MISMATCH BIT(26) #define MT_INPUTMODE_TOUCHSCREEN 0x02 #define MT_INPUTMODE_TOUCHPAD 0x03 @@ -235,6 +236,7 @@ static void mt_post_parse(struct mt_device *td, struct mt_application *app); #define MT_CLS_APPLE_TOUCHBAR 0x0114 #define MT_CLS_YOGABOOK9I 0x0115 #define MT_CLS_EGALAX_P80H84 0x0116 +#define MT_CLS_ASUS_ROG_Z13_FOLIO 0x0117 #define MT_CLS_SIS 0x0457 #define MT_DEFAULT_MAXCONTACT 10 @@ -405,6 +407,16 @@ static const struct mt_class mt_classes[] = { .quirks = MT_QUIRK_ALWAYS_VALID | MT_QUIRK_CONTACT_CNT_ACCURATE | MT_QUIRK_ASUS_CUSTOM_UP }, + { .name = MT_CLS_ASUS_ROG_Z13_FOLIO, + .quirks = MT_QUIRK_ALWAYS_VALID | + MT_QUIRK_IGNORE_DUPLICATES | + MT_QUIRK_HOVERING | + MT_QUIRK_CONTACT_CNT_ACCURATE | + MT_QUIRK_STICKY_FINGERS | + MT_QUIRK_WIN8_PTP_BUTTONS | + MT_QUIRK_CONFIDENCE | + MT_QUIRK_IGNORE_FEATURE_ID_MISMATCH, + .export_all_inputs = true }, { .name = MT_CLS_VTL, .quirks = MT_QUIRK_ALWAYS_VALID | MT_QUIRK_CONTACT_CNT_ACCURATE | @@ -507,6 +519,7 @@ static const struct attribute_group mt_attribute_group = { static void mt_get_feature(struct hid_device *hdev, struct hid_report *report) { + struct mt_device *td = hid_get_drvdata(hdev); int ret; u32 size = hid_report_len(report); u8 *buf; @@ -528,8 +541,14 @@ static void mt_get_feature(struct hid_device *hdev, struct hid_report *report) dev_warn(&hdev->dev, "failed to fetch feature %d\n", report->id); } else { - /* The report ID in the request and the response should match */ - if (report->id != buf[0]) { + /* + * The report ID in the request and the response should match. + * Some firmware (e.g. the ASUS ROG Z13 Folio + * touchpad) returns a mismatched ID on this specific fetch; + * tolerate it only for devices explicitly flagged as such. + */ + if (report->id != buf[0] && + !(td->mtclass.quirks & MT_QUIRK_IGNORE_FEATURE_ID_MISMATCH)) { hid_err(hdev, "Returned feature report did not match the request\n"); goto free; } @@ -2230,7 +2249,7 @@ static void mt_remove(struct hid_device *hdev) { struct mt_device *td = hid_get_drvdata(hdev); - timer_delete_sync(&td->release_timer); + timer_shutdown_sync(&td->release_timer); sysfs_remove_group(&hdev->dev.kobj, &mt_attribute_group); hid_hw_stop(hdev); @@ -2715,6 +2734,12 @@ static const struct hid_device_id mt_devices[] = { HID_DEVICE(BUS_I2C, HID_GROUP_MULTITOUCH_WIN_8, I2C_VENDOR_ID_HANTICK, I2C_PRODUCT_ID_HANTICK_5288) }, + /* Asus ROG Flow Z13 (2025) GZ302EA keyboard-cover touchpad */ + { .driver_data = MT_CLS_ASUS_ROG_Z13_FOLIO, + HID_DEVICE(BUS_USB, HID_GROUP_MULTITOUCH_WIN_8, + USB_VENDOR_ID_ASUSTEK, + USB_DEVICE_ID_ASUSTEK_ROG_Z13_FOLIO) }, + /* Generic MT device */ { HID_DEVICE(HID_BUS_ANY, HID_GROUP_MULTITOUCH, HID_ANY_ID, HID_ANY_ID) }, diff --git a/drivers/hid/hid-oxp.c b/drivers/hid/hid-oxp.c index d2ded6b08ce9..1e691ebc1199 100644 --- a/drivers/hid/hid-oxp.c +++ b/drivers/hid/hid-oxp.c @@ -1552,9 +1552,9 @@ static int oxp_hid_probe(struct hid_device *hdev, static void oxp_hid_remove(struct hid_device *hdev) { - cancel_delayed_work(&drvdata.oxp_rgb_queue); - cancel_delayed_work(&drvdata.oxp_btn_queue); - cancel_delayed_work(&drvdata.oxp_mcu_init); + cancel_delayed_work_sync(&drvdata.oxp_rgb_queue); + cancel_delayed_work_sync(&drvdata.oxp_btn_queue); + cancel_delayed_work_sync(&drvdata.oxp_mcu_init); hid_hw_close(hdev); hid_hw_stop(hdev); } diff --git a/drivers/hid/hid-quirks.c b/drivers/hid/hid-quirks.c index 8a0b51d47040..96a36c5ba04f 100644 --- a/drivers/hid/hid-quirks.c +++ b/drivers/hid/hid-quirks.c @@ -186,6 +186,7 @@ static const struct hid_device_id hid_quirks[] = { { HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_X52_2), HID_QUIRK_INCREMENT_USAGE_ON_DUPLICATE }, { HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_X52_PRO), HID_QUIRK_INCREMENT_USAGE_ON_DUPLICATE }, { HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_X65), HID_QUIRK_INCREMENT_USAGE_ON_DUPLICATE }, + { HID_USB_DEVICE(USB_VENDOR_ID_SDINNOVATION, USB_DEVICE_ID_SDINNOVATION_GAMING_KBD), HID_QUIRK_ALWAYS_POLL }, { HID_USB_DEVICE(USB_VENDOR_ID_SEMICO, USB_DEVICE_ID_SEMICO_USB_KEYKOARD2), HID_QUIRK_NO_INIT_REPORTS }, { HID_USB_DEVICE(USB_VENDOR_ID_SEMICO, USB_DEVICE_ID_SEMICO_USB_KEYKOARD), HID_QUIRK_NO_INIT_REPORTS }, { HID_USB_DEVICE(USB_VENDOR_ID_SENNHEISER, USB_DEVICE_ID_SENNHEISER_BTD500USB), HID_QUIRK_NOGET }, @@ -425,13 +426,14 @@ static const struct hid_device_id hid_have_special_driver[] = { #endif #if IS_ENABLED(CONFIG_HID_ELECOM) { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_BM084) }, - { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XGL20DLBK) }, + { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XGL20DLBK) }, { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_HT1MRBK_01AC) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3URBK_00FB) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3URBK_018F) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3DRBK_00FC) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT3DRBK_018C) }, - { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT4DRBK) }, + { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT4DRBK_00FD) }, + { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_XT4DRBK_018E) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_DT1URBK) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_DT1DRBK) }, { HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_M_DT2DRBK) }, @@ -751,6 +753,7 @@ static const struct hid_device_id hid_have_special_driver[] = { #if IS_ENABLED(CONFIG_HID_STEELSERIES) { HID_USB_DEVICE(USB_VENDOR_ID_STEELSERIES, USB_DEVICE_ID_STEELSERIES_SRWS1) }, { HID_USB_DEVICE(USB_VENDOR_ID_STEELSERIES, USB_DEVICE_ID_STEELSERIES_ARCTIS_1_X) }, + { HID_USB_DEVICE(USB_VENDOR_ID_STEELSERIES, USB_DEVICE_ID_STEELSERIES_ARCTIS_7_2018) }, { HID_USB_DEVICE(USB_VENDOR_ID_STEELSERIES, USB_DEVICE_ID_STEELSERIES_ARCTIS_9) }, { HID_USB_DEVICE(USB_VENDOR_ID_STEELSERIES, USB_DEVICE_ID_STEELSERIES_ARCTIS_NOVA_5_X) }, { HID_USB_DEVICE(USB_VENDOR_ID_STEELSERIES, USB_DEVICE_ID_STEELSERIES_ARCTIS_NOVA_7) }, diff --git a/drivers/hid/hid-roccat.c b/drivers/hid/hid-roccat.c index 4f15eb951039..5deb6da8d4f7 100644 --- a/drivers/hid/hid-roccat.c +++ b/drivers/hid/hid-roccat.c @@ -344,8 +344,6 @@ int roccat_connect(const struct class *klass, struct hid_device *hid, int report return temp; } - mutex_unlock(&devices_lock); - init_waitqueue_head(&device->wait); INIT_LIST_HEAD(&device->readers); mutex_init(&device->readers_lock); @@ -356,6 +354,7 @@ int roccat_connect(const struct class *klass, struct hid_device *hid, int report device->cbuf_end = 0; device->report_size = report_size; + mutex_unlock(&devices_lock); return minor; } EXPORT_SYMBOL_GPL(roccat_connect); @@ -369,15 +368,12 @@ void roccat_disconnect(int minor) mutex_lock(&devices_lock); device = devices[minor]; - mutex_unlock(&devices_lock); device->exist = 0; /* TODO exist maybe not needed */ device_destroy(device->dev->class, MKDEV(roccat_major, minor)); - mutex_lock(&devices_lock); devices[minor] = NULL; - mutex_unlock(&devices_lock); if (device->open) { hid_hw_close(device->hid); @@ -385,6 +381,8 @@ void roccat_disconnect(int minor) } else { roccat_free_device(device); } + + mutex_unlock(&devices_lock); } EXPORT_SYMBOL_GPL(roccat_disconnect); diff --git a/drivers/hid/hid-steelseries-arctis.c b/drivers/hid/hid-steelseries-arctis.c index 7a855b66730a..df61fa33d4c6 100644 --- a/drivers/hid/hid-steelseries-arctis.c +++ b/drivers/hid/hid-steelseries-arctis.c @@ -101,6 +101,19 @@ static int steelseries_arctis_1_request_status(struct hid_device *hdev) return steelseries_send_output_report(hdev, data, sizeof(data)); } +static int steelseries_arctis_7_2018_request_status(struct hid_device *hdev) +{ + const u8 connection[] = { 0x06, 0x14 }; + const u8 battery[] = { 0x06, 0x18 }; + int ret; + + ret = steelseries_send_output_report(hdev, connection, sizeof(connection)); + if (ret) + return ret; + + return steelseries_send_output_report(hdev, battery, sizeof(battery)); +} + static int steelseries_arctis_9_request_status(struct hid_device *hdev) { const u8 data[] = { 0x00, 0x20 }; @@ -152,6 +165,27 @@ static void steelseries_arctis_1_parse_status(struct steelseries_device *sd, sd->battery_capacity = data[3]; } +static void steelseries_arctis_7_2018_parse_status(struct steelseries_device *sd, + u8 *data, int size) +{ + if (size < 3 || data[0] != 0x06) + return; + + switch (data[1]) { + case 0x14: + /* 0x03 means that the headset is connected to the transmitter. */ + sd->headset_connected = data[2] == 0x03; + break; + case 0x18: + if (!sd->headset_connected) + break; + + /* The Arctis 7 sometimes overreports battery. Cap to 100. */ + sd->battery_capacity = steelseries_map_capacity(data[2], 0, 100); + break; + } +} + static void steelseries_arctis_9_parse_status(struct steelseries_device *sd, u8 *data, int size) { @@ -232,6 +266,13 @@ static const struct steelseries_device_info arctis_1_info = { .parse_status = steelseries_arctis_1_parse_status, }; +static const struct steelseries_device_info arctis_7_2018_info = { + .sync_interface = 5, + .capabilities = SS_CAP_BATTERY, + .request_status = steelseries_arctis_7_2018_request_status, + .parse_status = steelseries_arctis_7_2018_parse_status, +}; + static const struct steelseries_device_info arctis_9_info = { .sync_interface = 0, .capabilities = SS_CAP_BATTERY, @@ -653,6 +694,9 @@ static const struct hid_device_id steelseries_arctis_devices[] = { { HID_USB_DEVICE(USB_VENDOR_ID_STEELSERIES, USB_DEVICE_ID_STEELSERIES_ARCTIS_1_X), .driver_data = (unsigned long)&arctis_1_info }, + { HID_USB_DEVICE(USB_VENDOR_ID_STEELSERIES, + USB_DEVICE_ID_STEELSERIES_ARCTIS_7_2018), + .driver_data = (unsigned long)&arctis_7_2018_info }, { HID_USB_DEVICE(USB_VENDOR_ID_STEELSERIES, USB_DEVICE_ID_STEELSERIES_ARCTIS_9), .driver_data = (unsigned long)&arctis_9_info }, diff --git a/drivers/hid/hid-universal-pidff.c b/drivers/hid/hid-universal-pidff.c index 60180467a0bf..f1854cfe42d0 100644 --- a/drivers/hid/hid-universal-pidff.c +++ b/drivers/hid/hid-universal-pidff.c @@ -187,6 +187,7 @@ static const struct hid_device_id universal_pidff_devices[] = { { HID_USB_DEVICE(USB_VENDOR_ID_ASETEK, USB_DEVICE_ID_ASETEK_FORTE) }, { HID_USB_DEVICE(USB_VENDOR_ID_ASETEK, USB_DEVICE_ID_ASETEK_LA_PRIMA) }, { HID_USB_DEVICE(USB_VENDOR_ID_ASETEK, USB_DEVICE_ID_ASETEK_TONY_KANAAN) }, + { HID_USB_DEVICE(USB_VENDOR_ID_TURTLE_BEACH, USB_DEVICE_ID_TB_VELOCITYONE_RACE)}, {} }; MODULE_DEVICE_TABLE(hid, universal_pidff_devices); diff --git a/drivers/hid/hid-winwing.c b/drivers/hid/hid-winwing.c index 9cd25a77999e..19b92c2c6579 100644 --- a/drivers/hid/hid-winwing.c +++ b/drivers/hid/hid-winwing.c @@ -315,7 +315,8 @@ static void winwing_haptic_rumble_cb(struct work_struct *work) static int winwing_play_effect(struct input_dev *dev, void *context, struct ff_effect *effect) { - struct winwing_drv_data *data = (struct winwing_drv_data *) context; + struct hid_device *hdev = input_get_drvdata(dev); + struct winwing_drv_data *data = hid_get_drvdata(hdev); if (effect->type != FF_RUMBLE) return 0; @@ -342,7 +343,12 @@ static int winwing_init_ff(struct hid_device *hdev, struct hid_input *hidinput) input_set_capability(hidinput->input, EV_FF, FF_RUMBLE); - return input_ff_create_memless(hidinput->input, data, + /* + * input_ff_create_memless() takes ownership of the context pointer + * and frees it on teardown; do not hand it the devm-managed drvdata. + * winwing_play_effect() fetches it from the input device instead. + */ + return input_ff_create_memless(hidinput->input, NULL, winwing_play_effect); } diff --git a/drivers/hid/i2c-hid/i2c-hid-core.c b/drivers/hid/i2c-hid/i2c-hid-core.c index 0ff07fdab442..a61fd1de5203 100644 --- a/drivers/hid/i2c-hid/i2c-hid-core.c +++ b/drivers/hid/i2c-hid/i2c-hid-core.c @@ -123,9 +123,11 @@ static const struct i2c_hid_quirks { __u32 quirks; } i2c_hid_quirks[] = { { I2C_VENDOR_ID_HANTICK, I2C_PRODUCT_ID_HANTICK_5288, - I2C_HID_QUIRK_NO_IRQ_AFTER_RESET }, + I2C_HID_QUIRK_NO_IRQ_AFTER_RESET | I2C_HID_QUIRK_BAD_INPUT_SIZE }, { I2C_VENDOR_ID_ITE, I2C_DEVICE_ID_ITE_VOYO_WINPAD_A15, I2C_HID_QUIRK_NO_IRQ_AFTER_RESET }, + { USB_VENDOR_ID_ITE, I2C_DEVICE_ID_ITE_LENOVO_YOGA_SLIM_7X_G11_KEYBOARD, + I2C_HID_QUIRK_NO_IRQ_AFTER_RESET }, { I2C_VENDOR_ID_RAYDIUM, I2C_PRODUCT_ID_RAYDIUM_3118, I2C_HID_QUIRK_NO_IRQ_AFTER_RESET }, { USB_VENDOR_ID_ALPS_JP, HID_ANY_ID, diff --git a/drivers/hid/intel-thc-hid/intel-quicki2c/pci-quicki2c.c b/drivers/hid/intel-thc-hid/intel-quicki2c/pci-quicki2c.c index 0d2ad7bc3648..2537288b5026 100644 --- a/drivers/hid/intel-thc-hid/intel-quicki2c/pci-quicki2c.c +++ b/drivers/hid/intel-thc-hid/intel-quicki2c/pci-quicki2c.c @@ -606,7 +606,8 @@ static int quicki2c_alloc_report_buf(struct quicki2c_device *qcdev) */ max_report_len = max(le16_to_cpu(qcdev->dev_desc.max_input_len), SZ_4K); - qcdev->input_buf = devm_kzalloc(qcdev->dev, max_report_len, GFP_KERNEL); + qcdev->input_buf = devm_kzalloc(qcdev->dev, + HIDI2C_PACKET_LEN(max_report_len), GFP_KERNEL); if (!qcdev->input_buf) return -ENOMEM; @@ -616,7 +617,8 @@ static int quicki2c_alloc_report_buf(struct quicki2c_device *qcdev) max_report_len = max(le16_to_cpu(qcdev->dev_desc.max_output_len), max_report_len); - qcdev->report_buf = devm_kzalloc(qcdev->dev, max_report_len, GFP_KERNEL); + qcdev->report_buf = devm_kzalloc(qcdev->dev, + HIDI2C_PACKET_LEN(max_report_len), GFP_KERNEL); if (!qcdev->report_buf) return -ENOMEM; diff --git a/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c b/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c index 89226f5ce45e..ad676fde211b 100644 --- a/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c +++ b/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c @@ -552,7 +552,8 @@ static int quickspi_alloc_report_buf(struct quickspi_device *qsdev) max_input_len = max(le16_to_cpu(qsdev->dev_desc.rep_desc_len), le16_to_cpu(qsdev->dev_desc.max_input_len)); - qsdev->input_buf = devm_kzalloc(qsdev->dev, max_input_len, GFP_KERNEL); + qsdev->input_buf = devm_kzalloc(qsdev->dev, + HIDSPI_INPUT_BODY_SIZE(max_input_len), GFP_KERNEL); if (!qsdev->input_buf) return -ENOMEM; diff --git a/drivers/hid/wacom_sys.c b/drivers/hid/wacom_sys.c index 0eafa483b7f7..40770affdbde 100644 --- a/drivers/hid/wacom_sys.c +++ b/drivers/hid/wacom_sys.c @@ -113,8 +113,9 @@ static int wacom_wac_pen_serial_enforce(struct hid_device *hdev, /* Queue events which have invalid tool type or serial number */ for (i = 0; i < report->maxfield; i++) { - for (j = 0; j < report->field[i]->maxusage; j++) { - struct hid_field *field = report->field[i]; + struct hid_field *field = report->field[i]; + + for (j = 0; j < field->report_count; j++) { struct hid_usage *usage = &field->usage[j]; unsigned int equivalent_usage = wacom_equivalent_usage(usage->hid); unsigned int offset; diff --git a/drivers/i2c/busses/i2c-at91-master.c b/drivers/i2c/busses/i2c-at91-master.c index 68238cc8aee0..e0c3de5e8203 100644 --- a/drivers/i2c/busses/i2c-at91-master.c +++ b/drivers/i2c/busses/i2c-at91-master.c @@ -921,8 +921,14 @@ int at91_twi_probe_master(struct platform_device *pdev, at91_calc_twi_clock(dev); rc = at91_init_twi_recovery_info(pdev, dev); - if (rc == -EPROBE_DEFER) + if (rc == -EPROBE_DEFER) { + /* + * The DMA channels were already set up above, release them + * before deferring the probe so they are not leaked. + */ + at91_twi_dma_release(dev); return rc; + } dev->adapter.algo = &at91_twi_algorithm; dev->adapter.quirks = &at91_twi_quirks; diff --git a/drivers/i2c/busses/i2c-qcom-geni.c b/drivers/i2c/busses/i2c-qcom-geni.c index e9e41a174c20..6a39e21d5ed4 100644 --- a/drivers/i2c/busses/i2c-qcom-geni.c +++ b/drivers/i2c/busses/i2c-qcom-geni.c @@ -82,6 +82,9 @@ enum geni_i2c_err_code { #define XFER_TIMEOUT HZ #define RST_TIMEOUT HZ +#define GENI_SE_CLK_32MHZ (32 * HZ_PER_MHZ) +#define GENI_SE_CLK_19P2MHZ 19200000UL + struct geni_i2c_desc { bool no_dma_support; unsigned int tx_fifo_depth; @@ -127,6 +130,7 @@ struct geni_i2c_dev { spinlock_t lock; u32 clk_freq_out; const struct geni_i2c_clk_fld *clk_fld; + u32 clk_idx; void *dma_buf; size_t xfer_len; dma_addr_t dma_addr; @@ -197,19 +201,44 @@ static const struct geni_i2c_clk_fld geni_i2c_clk_map_32mhz[] = { static int geni_i2c_clk_map_idx(struct geni_i2c_dev *gi2c) { const struct geni_i2c_clk_fld *itr; + unsigned long res_freq; - if (clk_get_rate(gi2c->se.clk) == 32 * HZ_PER_MHZ) + /* + * Frequency counter tables are calibrated for a specific source + * clock frequency and are not valid for any multiple of it + * (e.g. 64 MHz, 128 MHz). + * Use exact=true and verify res_freq matches req_freq literally + * to reject harmonics: a 64 MHz clock that divides evenly to + * 32 MHz would pass exact matching but produce double the intended + * I2C frequency with these counter values. + */ + if (!geni_se_clk_freq_match(&gi2c->se, GENI_SE_CLK_32MHZ, + &gi2c->clk_idx, &res_freq, true) && + res_freq == GENI_SE_CLK_32MHZ) { itr = geni_i2c_clk_map_32mhz; - else + } else if (!geni_se_clk_freq_match(&gi2c->se, GENI_SE_CLK_19P2MHZ, + &gi2c->clk_idx, &res_freq, true) && + res_freq == GENI_SE_CLK_19P2MHZ) { itr = geni_i2c_clk_map_19p2mhz; + } else { + dev_err(gi2c->se.dev, + "Unsupported SE source clock: must be exactly 32 MHz or 19.2 MHz\n"); + return -EINVAL; + } while (itr->clk_freq_out != 0) { if (itr->clk_freq_out == gi2c->clk_freq_out) { gi2c->clk_fld = itr; + dev_dbg(gi2c->se.dev, + "I2C clk selected: freq: %u Hz, clk_idx: %u\n", + gi2c->clk_freq_out, gi2c->clk_idx); return 0; } itr++; } + + dev_err(gi2c->se.dev, "Unsupported I2C output frequency %u Hz\n", gi2c->clk_freq_out); + return -EINVAL; } @@ -219,7 +248,7 @@ static int qcom_geni_i2c_conf(struct geni_se *se, unsigned long freq) const struct geni_i2c_clk_fld *itr = gi2c->clk_fld; u32 val; - writel_relaxed(0, gi2c->se.base + SE_GENI_CLK_SEL); + writel_relaxed(gi2c->clk_idx, gi2c->se.base + SE_GENI_CLK_SEL); val = (itr->clk_div << CLK_DIV_SHFT) | SER_CLK_EN; writel_relaxed(val, gi2c->se.base + GENI_SER_M_CLK_CFG); @@ -1111,8 +1140,7 @@ static int geni_i2c_resources_init(struct geni_se *se) ret = geni_i2c_clk_map_idx(gi2c); if (ret) - return dev_err_probe(gi2c->se.dev, ret, "Invalid clk frequency %d Hz\n", - gi2c->clk_freq_out); + return ret; return geni_icc_set_bw_ab(&gi2c->se, GENI_DEFAULT_BW, GENI_DEFAULT_BW, Bps_to_icc(gi2c->clk_freq_out)); diff --git a/drivers/i2c/busses/i2c-xiic.c b/drivers/i2c/busses/i2c-xiic.c index 3e7735e1dae0..5e397a7e63f6 100644 --- a/drivers/i2c/busses/i2c-xiic.c +++ b/drivers/i2c/busses/i2c-xiic.c @@ -73,6 +73,9 @@ enum i2c_scl_freq { * @prev_msg_tx: Previous message is Tx * @quirks: To hold platform specific bug info * @smbus_block_read: Flag to handle block read + * @smbus_actual_len: Valid byte count (length + payload + optional PEC) of a + * padded SMBus block read. msg->len is trimmed to this on completion. + * Zero when no trimming is needed. * @input_clk: Input clock to I2C controller * @i2c_clk: I2C SCL frequency * @atomic: Mode of transfer @@ -98,6 +101,7 @@ struct xiic_i2c { bool prev_msg_tx; u32 quirks; bool smbus_block_read; + unsigned int smbus_actual_len; unsigned long input_clk; unsigned int i2c_clk; bool atomic; @@ -539,30 +543,38 @@ static void xiic_smbus_block_read_setup(struct xiic_i2c *i2c) /* Check if received length is valid */ if (rxmsg_len <= I2C_SMBUS_BLOCK_MAX) { + unsigned int pec_len = i2c->rx_msg->len - 1; + /* Set Receive fifo depth */ - if (rxmsg_len > IIC_RX_FIFO_DEPTH) { + if (rxmsg_len + pec_len > IIC_RX_FIFO_DEPTH) { /* - * When Rx msg len greater than or equal to Rx fifo capacity - * Receive fifo depth should set to Rx fifo capacity minus 1 + * Trailing bytes (payload plus any PEC) exceed Rx FIFO + * capacity, so drain in chunks. This also keeps the + * else branch below from pushing rfd_set past the + * 4-bit XIIC_RFD_REG_OFFSET field. */ rfd_set = IIC_RX_FIFO_DEPTH - 1; - i2c->rx_msg->len = rxmsg_len + 1; - } else if ((rxmsg_len == 1) || - (rxmsg_len == 0)) { + i2c->rx_msg->len = rxmsg_len + 1 + pec_len; + } else if (1 + rxmsg_len + pec_len < SMBUS_BLOCK_READ_MIN_LEN) { /* - * Minimum of 3 bytes required to exit cleanly. 1 byte - * already received, Second byte is being received. Have - * to set NACK in read_rx before receiving the last byte + * The HW needs SMBUS_BLOCK_READ_MIN_LEN bytes on the + * bus to exit cleanly: by the time the ISR reads the + * length byte the second byte is already being clocked + * in, too late to NACK. Pad the drain target and record + * the real length, trimmed back on completion so the + * PEC check sees the right byte. */ rfd_set = 0; i2c->rx_msg->len = SMBUS_BLOCK_READ_MIN_LEN; + i2c->smbus_actual_len = 1 + rxmsg_len + pec_len; } else { /* - * When Rx msg len less than Rx fifo capacity - * Receive fifo depth should set to Rx msg len minus 2 + * All trailing bytes fit in the Rx FIFO. Defer RX_FULL + * until every one of them is buffered, so the drain + * takes xiic_read_rx()'s bytes_rem == 0 path. */ - rfd_set = rxmsg_len - 2; - i2c->rx_msg->len = rxmsg_len + 1; + rfd_set = rxmsg_len + pec_len - 1; + i2c->rx_msg->len = rxmsg_len + 1 + pec_len; } xiic_setreg8(i2c, XIIC_RFD_REG_OFFSET, rfd_set); @@ -575,6 +587,16 @@ static void xiic_smbus_block_read_setup(struct xiic_i2c *i2c) dev_err(i2c->adap.dev.parent, "smbus_block_read Invalid msg length\n"); } +/* + * Undo the setup-time padding of a short SMBus block read before rx_msg is + * cleared, so the PEC check sees the right byte. + */ +static void xiic_smbus_trim_len(struct xiic_i2c *i2c) +{ + if (i2c->rx_msg && i2c->smbus_actual_len) + i2c->rx_msg->len = i2c->smbus_actual_len; +} + static void xiic_read_rx(struct xiic_i2c *i2c) { u8 bytes_in_fifo, cr = 0, bytes_to_read = 0; @@ -797,6 +819,8 @@ static irqreturn_t xiic_process(int irq, void *dev_id) xiic_read_rx(i2c); if (xiic_rx_space(i2c) == 0) { + xiic_smbus_trim_len(i2c); + /* this is the last part of the message */ i2c->rx_msg = NULL; @@ -865,8 +889,11 @@ static irqreturn_t xiic_process(int irq, void *dev_id) if (i2c->tx_msg && i2c->smbus_block_read) { i2c->smbus_block_read = false; - /* Set requested message len=1 to indicate STATE_DONE */ - i2c->tx_msg->len = 1; + /* + * Drive xiic_tx_space() to 0 to signal STATE_DONE + * without truncating the rx_msg length. + */ + i2c->tx_pos = i2c->tx_msg->len; } if (!i2c->tx_msg) @@ -938,6 +965,7 @@ static void xiic_recv_atomic(struct xiic_i2c *i2c) return; } + xiic_smbus_trim_len(i2c); i2c->rx_msg = NULL; xiic_irq_clr_en(i2c, XIIC_INTR_TX_ERROR_MASK); @@ -955,6 +983,9 @@ static void xiic_start_recv(struct xiic_i2c *i2c) u8 cr = 0, rfd_set = 0; struct i2c_msg *msg = i2c->rx_msg = i2c->tx_msg; + /* A stale value from an aborted block read would truncate this msg. */ + i2c->smbus_actual_len = 0; + if (!i2c->atomic) dev_dbg(i2c->adap.dev.parent, "%s entry, ISR: 0x%x, CR: 0x%x\n", __func__, xiic_getreg32(i2c, XIIC_IISR_OFFSET), diff --git a/drivers/iio/accel/kionix-kx022a.c b/drivers/iio/accel/kionix-kx022a.c index 02dd1db7a646..9ff70a373141 100644 --- a/drivers/iio/accel/kionix-kx022a.c +++ b/drivers/iio/accel/kionix-kx022a.c @@ -649,7 +649,7 @@ static int kx022a_read_raw(struct iio_dev *idev, if (ret) return ret; - if ((regval & KX022A_MASK_ODR) > + if ((regval & KX022A_MASK_ODR) >= ARRAY_SIZE(kx022a_accel_samp_freq_table)) { dev_err(data->dev, "Invalid ODR\n"); return -EINVAL; @@ -980,26 +980,44 @@ static int kx022a_fifo_enable(struct kx022a_data *data) guard(mutex)(&data->mutex); ret = __kx022a_turn_on_off(data, false); if (ret) - return ret; + goto err_free_out; /* Update watermark to HW */ ret = kx022a_fifo_set_wmi(data); if (ret) - return ret; + goto err_turn_on_out; /* Enable buffer */ ret = regmap_set_bits(data->regmap, data->chip_info->buf_cntl2, KX022A_MASK_BUF_EN); if (ret) - return ret; + goto err_turn_on_out; data->state |= KX022A_STATE_FIFO; ret = regmap_set_bits(data->regmap, data->ien_reg, KX022A_MASK_WMI); if (ret) - return ret; + goto err_buf_disable_out; - return __kx022a_turn_on_off(data, true); + ret = __kx022a_turn_on_off(data, true); + if (ret) + goto err_wmi_clear_out; + + return ret; + +err_wmi_clear_out: + regmap_clear_bits(data->regmap, data->ien_reg, + KX022A_MASK_WMI); +err_buf_disable_out: + regmap_clear_bits(data->regmap, data->chip_info->buf_cntl2, + KX022A_MASK_BUF_EN); + data->state &= ~KX022A_STATE_FIFO; +err_turn_on_out: + __kx022a_turn_on_off(data, true); +err_free_out: + kfree(data->fifo_buffer); + + return ret; } static int kx022a_buffer_postenable(struct iio_dev *idev) diff --git a/drivers/iio/accel/kxcjk-1013.c b/drivers/iio/accel/kxcjk-1013.c index 166fb786425f..8994c9e8e048 100644 --- a/drivers/iio/accel/kxcjk-1013.c +++ b/drivers/iio/accel/kxcjk-1013.c @@ -1030,7 +1030,7 @@ static int kxcjk1013_write_event_config(struct iio_dev *indio_dev, struct kxcjk1013_data *data = iio_priv(indio_dev); int ret; - if (state && data->ev_enable_state) + if (state == data->ev_enable_state) return 0; mutex_lock(&data->mutex); diff --git a/drivers/iio/accel/sca3000.c b/drivers/iio/accel/sca3000.c index a92a563f6a4b..8f1ac1d3305e 100644 --- a/drivers/iio/accel/sca3000.c +++ b/drivers/iio/accel/sca3000.c @@ -638,7 +638,7 @@ static int sca3000_write_raw_samp_freq(struct sca3000_state *st, int val) if (val == base_freq / 2) ctrlval |= SCA3000_REG_OUT_CTRL_BUF_DIV_2; - if (val == base_freq / 4) + else if (val == base_freq / 4) ctrlval |= SCA3000_REG_OUT_CTRL_BUF_DIV_4; else if (val != base_freq) return -EINVAL; diff --git a/drivers/iio/adc/ad4030.c b/drivers/iio/adc/ad4030.c index e97400a1aa48..877b06007396 100644 --- a/drivers/iio/adc/ad4030.c +++ b/drivers/iio/adc/ad4030.c @@ -746,14 +746,21 @@ static int ad4030_set_chan_calibbias(struct iio_dev *indio_dev, static int ad4030_set_avg_frame_len(struct iio_dev *dev, int avg_val) { struct ad4030_state *st = iio_priv(dev); - unsigned int avg_log2 = ilog2(avg_val); unsigned int last_avg_idx = ARRAY_SIZE(ad4030_average_modes) - 1; + unsigned int avg_log2; int freq_hz; int ret; - if (avg_val < 0 || avg_val > ad4030_average_modes[last_avg_idx]) + /* Reject unsupported modes */ + if (avg_val > ad4030_average_modes[last_avg_idx]) return -EINVAL; + /* Avoid invalid values for logarithm since it's undefined */ + if (avg_val < 1) + return -EINVAL; + + avg_log2 = ilog2(avg_val); + if (st->offload_trigger) { /* * The sample averaging and sampling frequency configurations diff --git a/drivers/iio/adc/ad7173.c b/drivers/iio/adc/ad7173.c index eb47175a0528..faebb7b34a15 100644 --- a/drivers/iio/adc/ad7173.c +++ b/drivers/iio/adc/ad7173.c @@ -775,9 +775,9 @@ static int ad7173_load_config(struct ad7173_state *st, return ad_sd_write_reg(&st->sd, AD7173_REG_FILTER(free_cfg_slot), 2, FIELD_PREP(AD7173_FILTER_SINC3_MAP, 0) | - FIELD_PREP(AD7173_FILTER_ENHFILT_MASK, - post_filter_enable) | FIELD_PREP(AD7173_FILTER_ENHFILTEN, + post_filter_enable) | + FIELD_PREP(AD7173_FILTER_ENHFILT_MASK, post_filter_select) | FIELD_PREP(AD7173_FILTER_ORDER, 0) | FIELD_PREP(AD7173_FILTER_ODR_MASK, diff --git a/drivers/iio/adc/ad_sigma_delta.c b/drivers/iio/adc/ad_sigma_delta.c index 1b410291da53..119ddc5a13e2 100644 --- a/drivers/iio/adc/ad_sigma_delta.c +++ b/drivers/iio/adc/ad_sigma_delta.c @@ -498,7 +498,6 @@ static int ad_sd_buffer_postenable(struct iio_dev *indio_dev) const struct iio_scan_type *scan_type = &indio_dev->channels[0].scan_type; struct spi_transfer *xfer = sigma_delta->sample_xfer; unsigned int i, slot, channel; - u8 *samples_buf; int ret; if (sigma_delta->num_slots == 1) { @@ -530,7 +529,7 @@ static int ad_sd_buffer_postenable(struct iio_dev *indio_dev) xfer[1].bits_per_word = scan_type->realbits; xfer[1].len = spi_bpw_to_bytes(scan_type->realbits); } else { - unsigned int samples_buf_size, scan_size; + unsigned int scan_size; if (sigma_delta->active_slots > 1) { ret = ad_sigma_delta_append_status(sigma_delta, true); @@ -538,17 +537,6 @@ static int ad_sd_buffer_postenable(struct iio_dev *indio_dev) return ret; } - samples_buf_size = - ALIGN(slot * BITS_TO_BYTES(scan_type->storagebits), - sizeof(s64)); - samples_buf_size += sizeof(s64); - samples_buf = devm_krealloc(&sigma_delta->spi->dev, - sigma_delta->samples_buf, - samples_buf_size, GFP_KERNEL); - if (!samples_buf) - return -ENOMEM; - - sigma_delta->samples_buf = samples_buf; scan_size = BITS_TO_BYTES(scan_type->realbits + scan_type->shift); /* For 24-bit data, there is an extra byte of padding. */ xfer[1].rx_buf = &sigma_delta->rx_buf[scan_size == 3 ? 1 : 0]; @@ -855,6 +843,23 @@ int devm_ad_sd_setup_buffer_and_trigger(struct device *dev, struct iio_dev *indi indio_dev->setup_ops = &ad_sd_buffer_setup_ops; } else { + const struct iio_scan_type *scan_type = + &indio_dev->channels[0].scan_type; + unsigned int samples_buf_size; + + /* + * Worst-case size: all sequencer slots can be active, capped + * at num_slots by ad_sd_validate_scan_mask(). + */ + samples_buf_size = + ALIGN(sigma_delta->num_slots * + BITS_TO_BYTES(scan_type->storagebits), + sizeof(s64)); + samples_buf_size += sizeof(s64); + sigma_delta->samples_buf = devm_kzalloc(dev, samples_buf_size, GFP_KERNEL); + if (!sigma_delta->samples_buf) + return -ENOMEM; + ret = devm_iio_triggered_buffer_setup(dev, indio_dev, &iio_pollfunc_store_time, &ad_sd_trigger_handler, diff --git a/drivers/iio/adc/ade9000.c b/drivers/iio/adc/ade9000.c index c6c3ea953fea..4fc0eb7e70a8 100644 --- a/drivers/iio/adc/ade9000.c +++ b/drivers/iio/adc/ade9000.c @@ -219,7 +219,7 @@ #define ADE9000_ST1_ERROR2_BIT BIT(30) #define ADE9000_ST1_ERROR3_BIT BIT(31) #define ADE9000_ST1_CROSSING_FIRST 6 -#define ADE9000_ST1_CROSSING_DEPTH 25 +#define ADE9000_ST1_CROSSING_DEPTH 26 #define ADE9000_WFB_TRG_DIP_BIT BIT(0) #define ADE9000_WFB_TRG_SWELL_BIT BIT(1) @@ -444,7 +444,7 @@ static const struct iio_chan_spec_ext_info ade9000_ext_info[] = { BIT(IIO_CHAN_INFO_CALIBSCALE), \ .event_spec = ade9000_current_events, \ .num_event_specs = ARRAY_SIZE(ade9000_current_events), \ - .scan_index = num, \ + .scan_index = num * 2, \ .indexed = 1, \ .scan_type = { \ .sign = 's', \ @@ -464,7 +464,7 @@ static const struct iio_chan_spec_ext_info ade9000_ext_info[] = { BIT(IIO_CHAN_INFO_FREQUENCY), \ .event_spec = ade9000_voltage_events, \ .num_event_specs = ARRAY_SIZE(ade9000_voltage_events), \ - .scan_index = num + 1, /* interleave with current channels */ \ + .scan_index = num * 2 + 1, /* interleave with current channels */ \ .indexed = 1, \ .scan_type = { \ .sign = 's', \ @@ -1647,8 +1647,9 @@ static int ade9000_setup_clkout(struct device *dev, struct ade9000_state *st) return 0; /* CLKOUT passes through CLKIN with divider of 1 */ - clkout_hw = devm_clk_hw_register_divider(dev, "clkout", __clk_get_name(st->clkin), - CLK_SET_RATE_PARENT, NULL, 0, 1, 0, NULL); + clkout_hw = devm_clk_hw_register_fixed_factor(dev, "clkout", + __clk_get_name(st->clkin), + CLK_SET_RATE_PARENT, 1, 1); if (IS_ERR(clkout_hw)) return dev_err_probe(dev, PTR_ERR(clkout_hw), "Failed to register clkout"); @@ -1705,18 +1706,6 @@ static int ade9000_probe(struct spi_device *spi) if (ret) return ret; - ret = ade9000_request_irq(dev, "irq0", ade9000_irq0_thread, indio_dev); - if (ret) - return ret; - - ret = ade9000_request_irq(dev, "irq1", ade9000_irq1_thread, indio_dev); - if (ret) - return ret; - - ret = ade9000_request_irq(dev, "dready", ade9000_dready_thread, indio_dev); - if (ret) - return ret; - /* External CMOS clock input (optional - crystal can be used instead) */ st->clkin = devm_clk_get_optional_enabled(dev, NULL); if (IS_ERR(st->clkin)) @@ -1736,6 +1725,19 @@ static int ade9000_probe(struct spi_device *spi) return dev_err_probe(&spi->dev, ret, "Failed to get and enable vdd regulator\n"); + /* Request IRQs only after powering the chip; the handlers do SPI access. */ + ret = ade9000_request_irq(dev, "irq0", ade9000_irq0_thread, indio_dev); + if (ret) + return ret; + + ret = ade9000_request_irq(dev, "irq1", ade9000_irq1_thread, indio_dev); + if (ret) + return ret; + + ret = ade9000_request_irq(dev, "dready", ade9000_dready_thread, indio_dev); + if (ret) + return ret; + indio_dev->channels = ade9000_channels; indio_dev->num_channels = ARRAY_SIZE(ade9000_channels); diff --git a/drivers/iio/adc/adi-axi-adc.c b/drivers/iio/adc/adi-axi-adc.c index 589f03618ecf..fc7cc92b35f0 100644 --- a/drivers/iio/adc/adi-axi-adc.c +++ b/drivers/iio/adc/adi-axi-adc.c @@ -714,6 +714,10 @@ static int adi_axi_adc_probe(struct platform_device *pdev) if (!st) return -ENOMEM; + ret = devm_mutex_init(dev, &st->lock); + if (ret) + return ret; + base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(base)) return PTR_ERR(base); diff --git a/drivers/iio/adc/aspeed_adc.c b/drivers/iio/adc/aspeed_adc.c index 6169ce28261e..452c30cf04f2 100644 --- a/drivers/iio/adc/aspeed_adc.c +++ b/drivers/iio/adc/aspeed_adc.c @@ -585,7 +585,9 @@ static int aspeed_adc_probe(struct platform_device *pdev) return dev_err_probe(dev, PTR_ERR(data->rst), "invalid or missing reset controller device tree entry"); - reset_control_deassert(data->rst); + ret = reset_control_deassert(data->rst); + if (ret) + return ret; ret = devm_add_action_or_reset(dev, aspeed_adc_reset_assert, data->rst); if (ret) diff --git a/drivers/iio/adc/axp288_adc.c b/drivers/iio/adc/axp288_adc.c index c8283279c477..c98863d612ad 100644 --- a/drivers/iio/adc/axp288_adc.c +++ b/drivers/iio/adc/axp288_adc.c @@ -207,6 +207,14 @@ static const struct dmi_system_id axp288_adc_ts_bias_override[] = { }, .driver_data = (void *)(uintptr_t)AXP288_ADC_TS_BIAS_80UA, }, + { + /* Haier HV103H */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Haier"), + DMI_MATCH(DMI_PRODUCT_NAME, "HV103H"), + }, + .driver_data = (void *)(uintptr_t)AXP288_ADC_TS_BIAS_80UA, + }, { } }; diff --git a/drivers/iio/adc/max1363.c b/drivers/iio/adc/max1363.c index 65a2d92bb112..629c53af6472 100644 --- a/drivers/iio/adc/max1363.c +++ b/drivers/iio/adc/max1363.c @@ -404,6 +404,14 @@ static int max1363_read_single_chan(struct iio_dev *indio_dev, data = rxbuf[0]; } + + /* + * Differential channels are bipolar and the device returns the sample + * in two's complement, so sign-extend it from the resolution bit. + */ + if (chan->differential) + data = sign_extend32(data, st->chip_info->bits - 1); + *val = data; return 0; diff --git a/drivers/iio/adc/pac1934.c b/drivers/iio/adc/pac1934.c index 23055405a6e0..de59dc27c440 100644 --- a/drivers/iio/adc/pac1934.c +++ b/drivers/iio/adc/pac1934.c @@ -1108,6 +1108,10 @@ static int pac1934_acpi_parse_channel_config(struct i2c_client *client, devm_kmemdup(dev, rez->package.elements[i].string.pointer, (size_t)rez->package.elements[i].string.length + 1, GFP_KERNEL); + if (!info->labels[idx]) { + ACPI_FREE(rez); + return -ENOMEM; + } info->labels[idx][rez->package.elements[i].string.length] = '\0'; info->shunts[idx] = rez->package.elements[i + 1].integer.value * 1000; info->active_channels[idx] = (info->shunts[idx] != 0); diff --git a/drivers/iio/adc/rohm-bd79124.c b/drivers/iio/adc/rohm-bd79124.c index ed5427288961..8091a138cca8 100644 --- a/drivers/iio/adc/rohm-bd79124.c +++ b/drivers/iio/adc/rohm-bd79124.c @@ -200,7 +200,7 @@ static int bd79124gpo_set_multiple(struct gpio_chip *gc, unsigned long *mask, if (ret) return ret; - if (all_gpos ^ *mask) { + if (*mask & ~all_gpos) { dev_dbg(data->dev, "Invalid mux config. Can't set value.\n"); return -EINVAL; @@ -381,6 +381,9 @@ static int bd79124_start_measurement(struct bd79124_data *data, int chan) /* See if already started */ ret = regmap_read(data->map, BD79124_REG_AUTO_CHANNELS, &val); + if (ret) + return ret; + if (val & BIT(chan)) return 0; @@ -420,11 +423,16 @@ static int bd79124_stop_measurement(struct bd79124_data *data, int chan) /* See if already stopped */ ret = regmap_read(data->map, BD79124_REG_AUTO_CHANNELS, &enabled_chans); + if (ret) + return ret; + if (!(enabled_chans & BIT(chan))) return 0; ret = regmap_clear_bits(data->map, BD79124_REG_SEQ_CFG, BD79124_MSK_SEQ_START); + if (ret) + return ret; /* Clear the channel from the measured channels */ enabled_chans &= ~BIT(chan); @@ -524,7 +532,7 @@ static int bd79124_enable_event(struct bd79124_data *data, return ret; if (dir == IIO_EV_DIR_RISING) { - limit = &data->alarm_f_limit[channel]; + limit = &data->alarm_r_limit[channel]; reg = BD79124_GET_HIGH_LIMIT_REG(channel); } else { limit = &data->alarm_f_limit[channel]; @@ -919,13 +927,13 @@ static int bd79124_chan_init(struct bd79124_data *data, int channel) { int ret; - ret = regmap_write(data->map, BD79124_GET_HIGH_LIMIT_REG(channel), - BD79124_HIGH_LIMIT_MAX); + ret = bd79124_write_int_to_reg(data, BD79124_GET_HIGH_LIMIT_REG(channel), + BD79124_HIGH_LIMIT_MAX); if (ret) return ret; - return regmap_write(data->map, BD79124_GET_LOW_LIMIT_REG(channel), - BD79124_LOW_LIMIT_MIN); + return bd79124_write_int_to_reg(data, BD79124_GET_LOW_LIMIT_REG(channel), + BD79124_LOW_LIMIT_MIN); } static int bd79124_get_gpio_pins(const struct iio_chan_spec *cs, int num_channels) diff --git a/drivers/iio/adc/stm32-adc.c b/drivers/iio/adc/stm32-adc.c index 5c6c06b269be..90f0e257e6d1 100644 --- a/drivers/iio/adc/stm32-adc.c +++ b/drivers/iio/adc/stm32-adc.c @@ -1608,11 +1608,16 @@ static int stm32_adc_read_raw(struct iio_dev *indio_dev, ret = stm32_adc_single_conv(indio_dev, chan, val); else ret = -EINVAL; - - if (mask == IIO_CHAN_INFO_PROCESSED) - *val = STM32_ADC_VREFINT_VOLTAGE * adc->vrefint.vrefint_cal / *val; - iio_device_release_direct(indio_dev); + if (ret < 0) + return ret; + + if (mask == IIO_CHAN_INFO_PROCESSED) { + if (*val == 0) + return -EINVAL; + *val = STM32_ADC_VREFINT_VOLTAGE * adc->vrefint.vrefint_cal / *val; + } + return ret; case IIO_CHAN_INFO_SCALE: @@ -2263,33 +2268,37 @@ static int stm32_adc_populate_int_ch(struct iio_dev *indio_dev, const char *ch_n for (i = 0; i < STM32_ADC_INT_CH_NB; i++) { if (!strncmp(stm32_adc_ic[i].name, ch_name, STM32_ADC_CH_SZ)) { + bool na; + /* Check internal channel availability */ switch (i) { case STM32_ADC_INT_CH_VDDCORE: - if (!adc->cfg->regs->or_vddcore.reg) - dev_warn(&indio_dev->dev, - "%s channel not available\n", ch_name); + na = !adc->cfg->regs->or_vddcore.reg; break; case STM32_ADC_INT_CH_VDDCPU: - if (!adc->cfg->regs->or_vddcpu.reg) - dev_warn(&indio_dev->dev, - "%s channel not available\n", ch_name); + na = !adc->cfg->regs->or_vddcpu.reg; break; case STM32_ADC_INT_CH_VDDQ_DDR: - if (!adc->cfg->regs->or_vddq_ddr.reg) - dev_warn(&indio_dev->dev, - "%s channel not available\n", ch_name); + na = !adc->cfg->regs->or_vddq_ddr.reg; break; case STM32_ADC_INT_CH_VREFINT: - if (!adc->cfg->regs->ccr_vref.reg) - dev_warn(&indio_dev->dev, - "%s channel not available\n", ch_name); + na = !adc->cfg->regs->ccr_vref.reg; break; case STM32_ADC_INT_CH_VBAT: - if (!adc->cfg->regs->ccr_vbat.reg) - dev_warn(&indio_dev->dev, - "%s channel not available\n", ch_name); + na = !adc->cfg->regs->ccr_vbat.reg; break; + default: + return -EINVAL; + } + + if (na) { + /* + * Channel label matches an internal STM32 ADC channel. + * Warn about it, as there's normally no restriction on the + * name but that's not among available internal channels. + */ + dev_warn(&indio_dev->dev, "no %s internal channel\n", ch_name); + return 0; } if (stm32_adc_ic[i].idx != STM32_ADC_INT_CH_VREFINT) { diff --git a/drivers/iio/adc/sun4i-gpadc-iio.c b/drivers/iio/adc/sun4i-gpadc-iio.c index 203459ca9907..ac5ba76b979e 100644 --- a/drivers/iio/adc/sun4i-gpadc-iio.c +++ b/drivers/iio/adc/sun4i-gpadc-iio.c @@ -638,10 +638,10 @@ static int sun4i_gpadc_probe(struct platform_device *pdev) * thermal because no thermal DT node is found. */ if (IS_ERR(info->tzd) && PTR_ERR(info->tzd) != -ENODEV) { - dev_err(&pdev->dev, - "could not register thermal sensor: %ld\n", - PTR_ERR(info->tzd)); - return PTR_ERR(info->tzd); + ret = dev_err_probe(&pdev->dev, + PTR_ERR(info->tzd), + "could not register thermal sensor\n"); + goto err_map; } } @@ -657,7 +657,6 @@ static int sun4i_gpadc_probe(struct platform_device *pdev) if (!info->no_irq && IS_ENABLED(CONFIG_THERMAL_OF)) iio_map_array_unregister(indio_dev); - pm_runtime_put(&pdev->dev); pm_runtime_disable(&pdev->dev); return ret; @@ -668,7 +667,6 @@ static void sun4i_gpadc_remove(struct platform_device *pdev) struct iio_dev *indio_dev = platform_get_drvdata(pdev); struct sun4i_gpadc_iio *info = iio_priv(indio_dev); - pm_runtime_put(&pdev->dev); pm_runtime_disable(&pdev->dev); if (!IS_ENABLED(CONFIG_THERMAL_OF)) diff --git a/drivers/iio/adc/xilinx-xadc-core.c b/drivers/iio/adc/xilinx-xadc-core.c index cab66bb8cc1c..66202b3a3abb 100644 --- a/drivers/iio/adc/xilinx-xadc-core.c +++ b/drivers/iio/adc/xilinx-xadc-core.c @@ -1401,13 +1401,14 @@ static int xadc_probe(struct platform_device *pdev) } if (irq > 0) { - ret = devm_request_irq(dev, irq, xadc->ops->interrupt_handler, - 0, dev_name(dev), indio_dev); + /* Clear out any work queued by the interrupt */ + ret = devm_add_action_or_reset(dev, xadc_cancel_delayed_work, + &xadc->zynq_unmask_work); if (ret) return ret; - ret = devm_add_action_or_reset(dev, xadc_cancel_delayed_work, - &xadc->zynq_unmask_work); + ret = devm_request_irq(dev, irq, xadc->ops->interrupt_handler, + 0, dev_name(dev), indio_dev); if (ret) return ret; } diff --git a/drivers/iio/buffer/industrialio-buffer-dmaengine.c b/drivers/iio/buffer/industrialio-buffer-dmaengine.c index 1bd82d3db5ea..31ed2e3e7f52 100644 --- a/drivers/iio/buffer/industrialio-buffer-dmaengine.c +++ b/drivers/iio/buffer/industrialio-buffer-dmaengine.c @@ -104,10 +104,13 @@ static int iio_dmaengine_buffer_submit_block(struct iio_dma_buffer_queue *queue, if (block->sg_table) { unsigned long flags; - sgl = block->sg_table->sgl; - nents = sg_nents_for_len(sgl, block->bytes_used); - if (nents < 0) - return nents; + /* + * Only the first sgt->nents entries carry a valid + * sg_dma_address()/sg_dma_len() pair as mapping the table may + * have coalesced entries, in which case nents is smaller than + * orig_nents. + */ + nents = block->sg_table->nents; vecs = kmalloc_objs(*vecs, nents, GFP_ATOMIC); if (!vecs) @@ -115,7 +118,8 @@ static int iio_dmaengine_buffer_submit_block(struct iio_dma_buffer_queue *queue, len_total = block->bytes_used; - for (i = 0; i < nents; i++) { + sgl = block->sg_table->sgl; + for (i = 0; i < nents && len_total; i++) { vecs[i].addr = sg_dma_address(sgl); vecs[i].len = min(sg_dma_len(sgl), len_total); len_total -= vecs[i].len; @@ -123,6 +127,8 @@ static int iio_dmaengine_buffer_submit_block(struct iio_dma_buffer_queue *queue, sgl = sg_next(sgl); } + nents = i; + if (block->cyclic) flags = DMA_PREP_REPEAT; else diff --git a/drivers/iio/cdc/ad7150.c b/drivers/iio/cdc/ad7150.c index 2f35c6d2f9ce..b36ac4e2dae8 100644 --- a/drivers/iio/cdc/ad7150.c +++ b/drivers/iio/cdc/ad7150.c @@ -636,11 +636,13 @@ static const struct i2c_device_id ad7150_id[] = { MODULE_DEVICE_TABLE(i2c, ad7150_id); static const struct of_device_id ad7150_of_match[] = { - { "adi,ad7150" }, - { "adi,ad7151" }, - { "adi,ad7156" }, + { .compatible = "adi,ad7150" }, + { .compatible = "adi,ad7151" }, + { .compatible = "adi,ad7156" }, { } }; +MODULE_DEVICE_TABLE(of, ad7150_of_match); + static struct i2c_driver ad7150_driver = { .driver = { .name = "ad7150", diff --git a/drivers/iio/common/inv_sensors/inv_sensors_timestamp.c b/drivers/iio/common/inv_sensors/inv_sensors_timestamp.c index 88a82d1370c5..8c9e81a77e68 100644 --- a/drivers/iio/common/inv_sensors/inv_sensors_timestamp.c +++ b/drivers/iio/common/inv_sensors/inv_sensors_timestamp.c @@ -167,11 +167,16 @@ void inv_sensors_timestamp_interrupt(struct inv_sensors_timestamp *ts, valid = inv_validate_period(ts, period); } - /* if interrupt interval is valid, sync with interrupt timestamp */ + /* + * If interrupt interval is valid, sync with interrupt timestamp. + * Otherwise, use estimated value while ensuring interrupt timestamp + * remains the maximum possible value. + */ + period = inv_align_timestamp_it(ts, sample_nb); if (valid) - ts->period = inv_align_timestamp_it(ts, sample_nb); + ts->period = period; else - ts->period = ts->mult * ts->chip_period.val; + ts->period = min(ts->mult * ts->chip_period.val, period); } EXPORT_SYMBOL_NS_GPL(inv_sensors_timestamp_interrupt, "IIO_INV_SENSORS_TIMESTAMP"); diff --git a/drivers/iio/dac/mcp47a1.c b/drivers/iio/dac/mcp47a1.c index 0bf994aa0e4f..3ed306f3060f 100644 --- a/drivers/iio/dac/mcp47a1.c +++ b/drivers/iio/dac/mcp47a1.c @@ -26,7 +26,7 @@ struct mcp47a1_data { int vref_mV; }; -static const int mcp47a1_raw_avail[] = { 0, 1, MCP47A1_MAX_STEPS - 1 }; +static const int mcp47a1_raw_avail[] = { 0, 1, MCP47A1_MAX_STEPS }; static const struct iio_chan_spec mcp47a1_channel = { .type = IIO_VOLTAGE, @@ -46,7 +46,7 @@ static int mcp47a1_write(struct iio_dev *indio_dev, switch (mask) { case IIO_CHAN_INFO_RAW: - if (!in_range(val, 0, MCP47A1_MAX_STEPS)) + if (!in_range(val, 0, MCP47A1_MAX_STEPS + 1)) return -EINVAL; return i2c_smbus_write_byte_data(data->client, MCP47A1_CMD_CODE, diff --git a/drivers/iio/dac/rohm-bd79703.c b/drivers/iio/dac/rohm-bd79703.c index 6e2e7787f8a4..127ccc2c4a22 100644 --- a/drivers/iio/dac/rohm-bd79703.c +++ b/drivers/iio/dac/rohm-bd79703.c @@ -73,6 +73,9 @@ static int bd79703_write_raw(struct iio_dev *idev, { struct bd79703_data *data = iio_priv(idev); + if (mask != IIO_CHAN_INFO_RAW) + return -EINVAL; + if (val < 0 || val >= 1 << BD79703_DAC_BITS) return -EINVAL; diff --git a/drivers/iio/frequency/adf4377.c b/drivers/iio/frequency/adf4377.c index 4dd19a9aa994..7385be62fa6a 100644 --- a/drivers/iio/frequency/adf4377.c +++ b/drivers/iio/frequency/adf4377.c @@ -988,10 +988,10 @@ static const struct clk_ops adf4377_clk_ops = { static int adf4377_clk_register(struct adf4377_state *st) { + struct clk_parent_data parent_data = { }; struct spi_device *spi = st->spi; struct device *dev = &spi->dev; - struct clk_init_data init; - struct clk_parent_data parent_data; + struct clk_init_data init = { }; int ret; if (!device_property_present(dev, "#clock-cells")) diff --git a/drivers/iio/frequency/admv1013.c b/drivers/iio/frequency/admv1013.c index c9baaab574e3..078ea3e5aa03 100644 --- a/drivers/iio/frequency/admv1013.c +++ b/drivers/iio/frequency/admv1013.c @@ -195,7 +195,7 @@ static int admv1013_read_raw(struct iio_dev *indio_dev, switch (info) { case IIO_CHAN_INFO_CALIBBIAS: - switch (chan->channel) { + switch (chan->channel2) { case IIO_MOD_I: addr = ADMV1013_REG_OFFSET_ADJUST_I; break; @@ -601,11 +601,14 @@ static int admv1013_probe(struct spi_device *spi) "failed to get the LO input clock\n"); st->nb.notifier_call = admv1013_freq_change; - ret = devm_clk_notifier_register(dev, st->clkin, &st->nb); + + ret = devm_mutex_init(dev, &st->lock); if (ret) return ret; - mutex_init(&st->lock); + ret = devm_clk_notifier_register(dev, st->clkin, &st->nb); + if (ret) + return ret; ret = admv1013_init(st, vcm_uv); if (ret) diff --git a/drivers/iio/gyro/adis16136.c b/drivers/iio/gyro/adis16136.c index ff335f81e9ae..b687588ada03 100644 --- a/drivers/iio/gyro/adis16136.c +++ b/drivers/iio/gyro/adis16136.c @@ -145,7 +145,7 @@ static int adis16136_debugfs_init(struct iio_dev *indio_dev) struct adis16136 *adis16136 = iio_priv(indio_dev); struct dentry *d = iio_get_debugfs_dentry(indio_dev); - debugfs_create_file_unsafe("serial_number", 0400, + debugfs_create_file("serial_number", 0400, d, adis16136, &adis16136_serial_fops); debugfs_create_file_unsafe("product_id", 0400, d, adis16136, &adis16136_product_id_fops); diff --git a/drivers/iio/health/max30102.c b/drivers/iio/health/max30102.c index c37316c86f14..aee96167f01e 100644 --- a/drivers/iio/health/max30102.c +++ b/drivers/iio/health/max30102.c @@ -290,9 +290,15 @@ static irqreturn_t max30102_interrupt_handler(int irq, void *private) { struct iio_dev *indio_dev = private; struct max30102_data *data = iio_priv(indio_dev); - unsigned int measurements = bitmap_weight(indio_dev->active_scan_mask, - iio_get_masklength(indio_dev)); - int ret, cnt = 0; + unsigned int measurements; + int ret, cnt; + + cnt = max30102_fifo_count(data); + if (cnt <= 0) + return IRQ_HANDLED; + + measurements = bitmap_weight(indio_dev->active_scan_mask, + iio_get_masklength(indio_dev)); mutex_lock(&data->lock); diff --git a/drivers/iio/imu/adis16400.c b/drivers/iio/imu/adis16400.c index 4842346f9f0c..8f664d055e9d 100644 --- a/drivers/iio/imu/adis16400.c +++ b/drivers/iio/imu/adis16400.c @@ -280,7 +280,7 @@ static void adis16400_debugfs_init(struct iio_dev *indio_dev) return; if (st->variant->flags & ADIS16400_HAS_SERIAL_NUMBER) - debugfs_create_file_unsafe("serial_number", 0400, + debugfs_create_file("serial_number", 0400, d, st, &adis16400_serial_number_fops); if (st->variant->flags & ADIS16400_HAS_PROD_ID) debugfs_create_file_unsafe("product_id", 0400, diff --git a/drivers/iio/imu/adis16480.c b/drivers/iio/imu/adis16480.c index 5187566c1876..d51786874e74 100644 --- a/drivers/iio/imu/adis16480.c +++ b/drivers/iio/imu/adis16480.c @@ -309,9 +309,9 @@ static void adis16480_debugfs_init(struct iio_dev *indio_dev) if (!IS_ENABLED(CONFIG_DEBUG_FS)) return; - debugfs_create_file_unsafe("firmware_revision", 0400, + debugfs_create_file("firmware_revision", 0400, d, adis16480, &adis16480_firmware_revision_fops); - debugfs_create_file_unsafe("firmware_date", 0400, + debugfs_create_file("firmware_date", 0400, d, adis16480, &adis16480_firmware_date_fops); debugfs_create_file_unsafe("serial_number", 0400, d, adis16480, &adis16480_serial_number_fops); diff --git a/drivers/iio/imu/inv_icm42607/inv_icm42607_core.c b/drivers/iio/imu/inv_icm42607/inv_icm42607_core.c index 190e998f7b8e..f4ef75da22c7 100644 --- a/drivers/iio/imu/inv_icm42607/inv_icm42607_core.c +++ b/drivers/iio/imu/inv_icm42607/inv_icm42607_core.c @@ -537,9 +537,8 @@ static int inv_icm42607_enable_vddio_reg(struct inv_icm42607_state *st) return 0; } -static void inv_icm42607_sensors_off(void *_data) +static int inv_icm42607_sensors_off(struct inv_icm42607_state *st) { - struct inv_icm42607_state *st = _data; const struct device *dev = regmap_get_device(st->map); int ret; @@ -552,6 +551,13 @@ static void inv_icm42607_sensors_off(void *_data) st->conf.accel.mode); if (ret) dev_err(dev, "Unable to turn off sensors\n"); + + return ret; +} + +static void inv_icm42607_sensors_off_action(void *data) +{ + inv_icm42607_sensors_off(data); } static void inv_icm42607_disable_vddio_reg(void *_data) @@ -619,7 +625,7 @@ int inv_icm42607_core_probe(struct regmap *regmap, * Ensure if sensors get turned on at some point, they're turned off * as part of teardown. */ - ret = devm_add_action_or_reset(dev, inv_icm42607_sensors_off, st); + ret = devm_add_action_or_reset(dev, inv_icm42607_sensors_off_action, st); if (ret) return ret; @@ -658,9 +664,8 @@ static int inv_icm42607_suspend(struct device *dev) return 0; } -static int inv_icm42607_resume(struct device *dev) +static int inv_icm42607_resume_core(struct inv_icm42607_state *st) { - struct inv_icm42607_state *st = dev_get_drvdata(dev); int ret; ret = inv_icm42607_enable_vddio_reg(st); @@ -669,9 +674,25 @@ static int inv_icm42607_resume(struct device *dev) /* Sync the regcache again after regulator shutdown. */ regcache_mark_dirty(st->map); - ret = regcache_sync(st->map); - if (ret) + + return regcache_sync(st->map); +} + +static int inv_icm42607_resume(struct device *dev) +{ + struct inv_icm42607_state *st = dev_get_drvdata(dev); + int ret; + + ret = inv_icm42607_resume_core(st); + if (ret) { + int rc; + + rc = pm_runtime_force_resume(dev); + if (rc) + dev_warn(dev, "Failed to restore runtime PM state: %d\n", rc); + return ret; + } return pm_runtime_force_resume(dev); } @@ -688,8 +709,7 @@ static int inv_icm42607_runtime_suspend(struct device *dev) * however the tradeoff is that an unused sensor won't be * turned off until the entire chip is no longer in use. */ - inv_icm42607_sensors_off(st); - return 0; + return inv_icm42607_sensors_off(st); } EXPORT_NS_GPL_DEV_PM_OPS(inv_icm42607_pm_ops, IIO_ICM42607) = { diff --git a/drivers/iio/industrialio-buffer.c b/drivers/iio/industrialio-buffer.c index 2c9ec93dff47..4294101810fd 100644 --- a/drivers/iio/industrialio-buffer.c +++ b/drivers/iio/industrialio-buffer.c @@ -1377,6 +1377,10 @@ EXPORT_SYMBOL_GPL(iio_update_buffers); void iio_disable_all_buffers(struct iio_dev *indio_dev) { + struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev); + + guard(mutex)(&iio_dev_opaque->mlock); + iio_disable_buffers(indio_dev); iio_buffer_deactivate_all(indio_dev); } @@ -2462,7 +2466,8 @@ int iio_push_to_buffers_with_ts_unaligned(struct iio_dev *indio_dev, bb = devm_krealloc(&indio_dev->dev, iio_dev_opaque->bounce_buffer, - indio_dev->scan_bytes, GFP_KERNEL); + indio_dev->scan_bytes, + GFP_KERNEL | __GFP_ZERO); if (!bb) return -ENOMEM; iio_dev_opaque->bounce_buffer = bb; diff --git a/drivers/iio/industrialio-trigger.c b/drivers/iio/industrialio-trigger.c index 17781c12bc85..76926fee0f84 100644 --- a/drivers/iio/industrialio-trigger.c +++ b/drivers/iio/industrialio-trigger.c @@ -509,6 +509,8 @@ static void iio_trig_release(struct device *device) struct iio_trigger *trig = to_iio_trigger(device); int i; + cancel_work_sync(&trig->reenable_work); + if (trig->subirq_base) { for (i = 0; i < CONFIG_IIO_CONSUMERS_PER_TRIGGER; i++) { irq_modify_status(trig->subirq_base + i, diff --git a/drivers/iio/light/gp2ap020a00f.c b/drivers/iio/light/gp2ap020a00f.c index 63591b7ecb89..b435dc5e8706 100644 --- a/drivers/iio/light/gp2ap020a00f.c +++ b/drivers/iio/light/gp2ap020a00f.c @@ -1497,6 +1497,7 @@ static int gp2ap020a00f_probe(struct i2c_client *client) iio_trigger_unregister(data->trig); error_free_irq: free_irq(client->irq, indio_dev); + irq_work_sync(&data->work); error_uninit_buffer: iio_triggered_buffer_cleanup(indio_dev); error_regulator_disable: @@ -1519,6 +1520,7 @@ static void gp2ap020a00f_remove(struct i2c_client *client) iio_device_unregister(indio_dev); iio_trigger_unregister(data->trig); free_irq(client->irq, indio_dev); + irq_work_sync(&data->work); iio_triggered_buffer_cleanup(indio_dev); regulator_disable(data->vled_reg); } diff --git a/drivers/iio/light/rohm-bu27034.c b/drivers/iio/light/rohm-bu27034.c index 28d111ac8c0a..92fca33387bb 100644 --- a/drivers/iio/light/rohm-bu27034.c +++ b/drivers/iio/light/rohm-bu27034.c @@ -137,6 +137,7 @@ static const struct iio_gain_sel_pair bu27034_gains[] = { #define BU27034_MEAS_MODE_200MS 2 #define BU27034_MEAS_MODE_400MS 4 +#define BU27034_INT_TIME_US_MIN (55 * USEC_PER_MSEC) static const struct iio_itime_sel_mul bu27034_itimes[] = { GAIN_SCALE_ITIME_US(400000, BU27034_MEAS_MODE_400MS, 8), GAIN_SCALE_ITIME_US(200000, BU27034_MEAS_MODE_200MS, 4), @@ -296,7 +297,7 @@ static int bu27034_get_gain(struct bu27034_data *data, int chan, int *gain) return 0; } -static int bu27034_get_int_time(struct bu27034_data *data) +static int bu27034_get_int_time(struct bu27034_data *data, int *itime) { int ret, sel; @@ -304,24 +305,30 @@ static int bu27034_get_int_time(struct bu27034_data *data) if (ret) return ret; - return iio_gts_find_int_time_by_sel(&data->gts, - sel & BU27034_MASK_MEAS_MODE); + ret = iio_gts_find_int_time_by_sel(&data->gts, + sel & BU27034_MASK_MEAS_MODE); + if (ret < 0) + return ret; + + *itime = ret; + + return 0; } static int _bu27034_get_scale(struct bu27034_data *data, int channel, int *val, int *val2) { - int gain, ret; + int gain, itime, ret; ret = bu27034_get_gain(data, channel, &gain); if (ret) return ret; - ret = bu27034_get_int_time(data); - if (ret < 0) + ret = bu27034_get_int_time(data, &itime); + if (ret) return ret; - return iio_gts_get_scale(&data->gts, gain, ret, val, val2); + return iio_gts_get_scale(&data->gts, gain, itime, val, val2); } static int bu27034_get_scale(struct bu27034_data *data, int channel, int *val, @@ -397,12 +404,10 @@ static int bu27034_try_set_int_time(struct bu27034_data *data, int time_us) int ret, int_time_old, i; guard(mutex)(&data->mutex); - ret = bu27034_get_int_time(data); - if (ret < 0) + ret = bu27034_get_int_time(data, &int_time_old); + if (ret) return ret; - int_time_old = ret; - if (!iio_gts_valid_time(&data->gts, time_us)) { dev_err(data->dev, "Unsupported integration time %u\n", time_us); @@ -415,7 +420,7 @@ static int bu27034_try_set_int_time(struct bu27034_data *data, int time_us) for (i = 0; i < numg; i++) { ret = bu27034_get_gain(data, gains[i].chan, &gains[i].old_gain); if (ret) - return 0; + return ret; ret = iio_gts_find_new_gain_by_old_gain_time(&data->gts, gains[i].old_gain, @@ -841,7 +846,7 @@ static int bu27034_meas_set(struct bu27034_data *data, bool en) static int bu27034_get_single_result(struct bu27034_data *data, int chan, int *val) { - int ret; + int ret, itime; if (chan < BU27034_CHAN_DATA0 || chan > BU27034_CHAN_DATA1) return -EINVAL; @@ -850,11 +855,11 @@ static int bu27034_get_single_result(struct bu27034_data *data, int chan, if (ret) return ret; - ret = bu27034_get_int_time(data); - if (ret < 0) + ret = bu27034_get_int_time(data, &itime); + if (ret) return ret; - msleep(ret / 1000); + msleep(itime / 1000); return bu27034_read_result(data, chan, val); } @@ -904,12 +909,10 @@ static int bu27034_calc_mlux(struct bu27034_data *data, __le16 *res, int *val) if (ret) return ret; - ret = bu27034_get_int_time(data); - if (ret < 0) + ret = bu27034_get_int_time(data, &meastime); + if (ret) return ret; - meastime = ret; - d1_d0_ratio_scaled = (unsigned int)ch1 * (unsigned int)gain0 * 100; helper64 = (u64)ch1 * (u64)gain0 * 100LLU; @@ -970,9 +973,9 @@ static int bu27034_read_raw(struct iio_dev *idev, switch (mask) { case IIO_CHAN_INFO_INT_TIME: *val = 0; - *val2 = bu27034_get_int_time(data); - if (*val2 < 0) - return *val2; + ret = bu27034_get_int_time(data, val2); + if (ret) + return ret; return IIO_VAL_INT_PLUS_MICRO; @@ -1157,11 +1160,20 @@ static int bu27034_buffer_thread(void *arg) { struct iio_dev *idev = arg; struct bu27034_data *data; - int wait_ms; + int wait_ms, ret; data = iio_priv(idev); - wait_ms = bu27034_get_int_time(data); + /* + * If reading the integration time fails, default to the minimum so we + * don't lose samples. This may waste CPU cycles, but as a hardening + * against theoretical, once-in-a-blue-moon error, this should be Ok. + */ + wait_ms = BU27034_INT_TIME_US_MIN; + ret = bu27034_get_int_time(data, &wait_ms); + if (ret) + dev_warn(data->dev, "Failed to get integration time\n"); + wait_ms /= 1000; wait_ms -= BU27034_MEAS_WAIT_PREMATURE_MS; diff --git a/drivers/iio/pressure/bmp280-core.c b/drivers/iio/pressure/bmp280-core.c index 1066792b9087..3940b8e61a06 100644 --- a/drivers/iio/pressure/bmp280-core.c +++ b/drivers/iio/pressure/bmp280-core.c @@ -839,7 +839,7 @@ static int bmp280_write_sampling_frequency(struct bmp280_data *data, int val, int val2) { const int (*avail)[2] = data->chip_info->sampling_freq_avail; - const int n = data->chip_info->num_sampling_freq_avail; + const int n = data->chip_info->num_sampling_freq_avail / 2; int ret, prev; int i; diff --git a/drivers/iio/pressure/rohm-bm1390.c b/drivers/iio/pressure/rohm-bm1390.c index 57941fb4a535..f9deb48e6a41 100644 --- a/drivers/iio/pressure/rohm-bm1390.c +++ b/drivers/iio/pressure/rohm-bm1390.c @@ -289,7 +289,7 @@ static int bm1390_read_data(struct bm1390_data *data, if (warn) dev_warn(data->dev, "Failed to stop measurement (%d)\n", warn); - return 0; + return ret; } static int bm1390_read_raw(struct iio_dev *idev, diff --git a/drivers/iio/proximity/aw96103.c b/drivers/iio/proximity/aw96103.c index 8fbb755dcae0..8352d51e5128 100644 --- a/drivers/iio/proximity/aw96103.c +++ b/drivers/iio/proximity/aw96103.c @@ -24,6 +24,7 @@ #define AW96103_BIN_VALID_DATA_OFFSET 64 #define AW96103_BIN_DATA_LEN_OFFSET 16 #define AW96103_BIN_DATA_REG_NUM_SIZE 4 +#define AW96103_BIN_REG_SIZE 6 #define AW96103_BIN_CHIP_TYPE_SIZE 8 #define AW96103_BIN_CHIP_TYPE_OFFSET 24 @@ -229,14 +230,27 @@ static const struct aw_chip_info aw_chip_info_tbl[] = { }, }; -static void aw96103_parsing_bin_file(struct aw_bin *bin) +static int aw96103_parsing_bin_file(struct aw_bin *bin) { + u32 data_len; + + if (bin->len < AW96103_BIN_VALID_DATA_OFFSET) + return -EINVAL; + + data_len = get_unaligned_le32(bin->data + AW96103_BIN_DATA_LEN_OFFSET); + if (data_len < AW96103_BIN_DATA_REG_NUM_SIZE) + return -EINVAL; + + bin->valid_data_len = data_len - AW96103_BIN_DATA_REG_NUM_SIZE; + if (bin->valid_data_len > bin->len - AW96103_BIN_VALID_DATA_OFFSET || + bin->valid_data_len % AW96103_BIN_REG_SIZE) + return -EINVAL; + bin->valid_data_addr = AW96103_BIN_VALID_DATA_OFFSET; - bin->valid_data_len = - *(unsigned int *)(bin->data + AW96103_BIN_DATA_LEN_OFFSET) - - AW96103_BIN_DATA_REG_NUM_SIZE; memcpy(bin->chip_type, bin->data + AW96103_BIN_CHIP_TYPE_OFFSET, AW96103_BIN_CHIP_TYPE_SIZE); + + return 0; } static const struct regmap_config aw96103_regmap_confg = { @@ -500,7 +514,7 @@ static int aw96103_bin_valid_loaded(struct aw96103 *aw96103, int ret; for (i = 0; i < aw_bin_data_s->valid_data_len; - i += 6, start_addr += 6) { + i += AW96103_BIN_REG_SIZE, start_addr += AW96103_BIN_REG_SIZE) { reg_addr = get_unaligned_le16(aw_bin_data_s->data + start_addr); reg_data = get_unaligned_le32(aw_bin_data_s->data + start_addr + 2); @@ -550,6 +564,8 @@ static int aw96103_para_loaded(struct aw96103 *aw96103) static int aw96103_cfg_all_loaded(const struct firmware *cont, struct aw96103 *aw96103) { + int ret; + if (!cont) return -EINVAL; @@ -561,7 +577,9 @@ static int aw96103_cfg_all_loaded(const struct firmware *cont, aw_bin->len = cont->size; memcpy(aw_bin->data, cont->data, cont->size); release_firmware(cont); - aw96103_parsing_bin_file(aw_bin); + ret = aw96103_parsing_bin_file(aw_bin); + if (ret) + return ret; return aw96103_bin_valid_loaded(aw96103, aw_bin); } diff --git a/drivers/iio/proximity/isl29501.c b/drivers/iio/proximity/isl29501.c index 95fb7238f678..a98a9975323c 100644 --- a/drivers/iio/proximity/isl29501.c +++ b/drivers/iio/proximity/isl29501.c @@ -226,7 +226,7 @@ static int isl29501_register_read(struct isl29501_private *isl29501, return ret; } -static u32 isl29501_register_write(struct isl29501_private *isl29501, +static int isl29501_register_write(struct isl29501_private *isl29501, enum isl29501_register_name name, u32 value) { diff --git a/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c b/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c index 400477b4c740..8a9ee21f2bf6 100644 --- a/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c +++ b/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c @@ -294,12 +294,14 @@ static int lidar_probe(struct i2c_client *client) ret = pm_runtime_set_active(&client->dev); if (ret) - goto error_unreg_buffer; + goto error_unreg_dev; pm_runtime_enable(&client->dev); pm_runtime_idle(&client->dev); return 0; +error_unreg_dev: + iio_device_unregister(indio_dev); error_unreg_buffer: iio_triggered_buffer_cleanup(indio_dev); diff --git a/drivers/iio/proximity/sx9324.c b/drivers/iio/proximity/sx9324.c index 13b4ef2896e3..255a36976ac4 100644 --- a/drivers/iio/proximity/sx9324.c +++ b/drivers/iio/proximity/sx9324.c @@ -222,7 +222,7 @@ static const struct iio_chan_spec_ext_info sx9324_channel_ext_info[] = { .scan_index = idx, \ .scan_type = { \ .sign = 's', \ - .realbits = 12, \ + .realbits = 16, \ .storagebits = 16, \ .endianness = IIO_BE, \ }, \ diff --git a/drivers/iio/proximity/vcnl3020.c b/drivers/iio/proximity/vcnl3020.c index 6b0d7a9038d9..768870127ff8 100644 --- a/drivers/iio/proximity/vcnl3020.c +++ b/drivers/iio/proximity/vcnl3020.c @@ -584,16 +584,26 @@ static irqreturn_t vcnl3020_handle_irq_thread(int irq, void *p) return IRQ_HANDLED; } - if (!(isr & VCNL_ICR_THRES_EN)) + if (!(isr & (VCNL_INT_TH_HI | VCNL_INT_TH_LOW))) return IRQ_NONE; - iio_push_event(indio_dev, - IIO_UNMOD_EVENT_CODE(IIO_PROXIMITY, 1, - IIO_EV_TYPE_THRESH, - IIO_EV_DIR_RISING), - iio_get_time_ns(indio_dev)); + if (isr & VCNL_INT_TH_HI) { + iio_push_event(indio_dev, + IIO_UNMOD_EVENT_CODE(IIO_PROXIMITY, 0, + IIO_EV_TYPE_THRESH, + IIO_EV_DIR_RISING), + iio_get_time_ns(indio_dev)); + } - rc = regmap_write(data->regmap, VCNL_ISR, isr & VCNL_ICR_THRES_EN); + if (isr & VCNL_INT_TH_LOW) { + iio_push_event(indio_dev, + IIO_UNMOD_EVENT_CODE(IIO_PROXIMITY, 0, + IIO_EV_TYPE_THRESH, + IIO_EV_DIR_FALLING), + iio_get_time_ns(indio_dev)); + } + + rc = regmap_write(data->regmap, VCNL_ISR, isr & (VCNL_INT_TH_HI | VCNL_INT_TH_LOW)); if (rc) dev_err(data->dev, "Error (%d) writing in reg (0x%x)\n", rc, VCNL_ISR); diff --git a/drivers/iio/proximity/vl53l0x-i2c.c b/drivers/iio/proximity/vl53l0x-i2c.c index 9fe14ceb8be7..173e69753df7 100644 --- a/drivers/iio/proximity/vl53l0x-i2c.c +++ b/drivers/iio/proximity/vl53l0x-i2c.c @@ -219,7 +219,10 @@ static int vl53l0x_read_raw(struct iio_dev *indio_dev, switch (mask) { case IIO_CHAN_INFO_RAW: + if (!iio_device_claim_direct(indio_dev)) + return -EBUSY; ret = vl53l0x_read_proximity(data, chan, val); + iio_device_release_direct(indio_dev); if (ret < 0) return ret; diff --git a/drivers/input/keyboard/atkbd.c b/drivers/input/keyboard/atkbd.c index 93650c98206d..b5ee5697a136 100644 --- a/drivers/input/keyboard/atkbd.c +++ b/drivers/input/keyboard/atkbd.c @@ -1938,6 +1938,14 @@ static const struct dmi_system_id atkbd_dmi_quirk_table[] __initconst = { }, .callback = atkbd_deactivate_fixup, }, + { + /* Lenovo IdeaPad Slim 3 15IWC11 (83RR) */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "83RR"), + }, + .callback = atkbd_deactivate_fixup, + }, { /* Xiaomi Book Pro 14 (TM2424) */ .matches = { diff --git a/drivers/input/mouse/synaptics.c b/drivers/input/mouse/synaptics.c index a8ce89d41faa..1ed9261bff2b 100644 --- a/drivers/input/mouse/synaptics.c +++ b/drivers/input/mouse/synaptics.c @@ -192,6 +192,7 @@ static const char * const smbus_pnp_ids[] = { "LEN2054", /* E480 */ "LEN2055", /* E580 */ "LEN2058", /* E490 */ + "LEN205b", /* T490 */ "LEN2068", /* T14 Gen 1 */ "SYN1221", /* TUXEDO InfinityBook Pro 14 v5 */ "SYN3003", /* HP EliteBook 850 G1 */ @@ -1839,8 +1840,10 @@ static int synaptics_setup_intertouch(struct psmouse *psmouse, return -ENXIO; } - /* Disable intertouch on known-broken board revisions */ - if (info->board_id == 2722) { + /* Disable intertouch on known-broken T440p board revisions */ + if (info->board_id == 2722 && + psmouse_matches_pnp_id(psmouse, + (const char * const []){"LEN0036", NULL})) { psmouse_info(psmouse, "Disabling intertouch for board id %u\n", info->board_id); diff --git a/drivers/input/serio/i8042-acpipnpio.h b/drivers/input/serio/i8042-acpipnpio.h index 54eb3687000a..1b1c5d4ac01c 100644 --- a/drivers/input/serio/i8042-acpipnpio.h +++ b/drivers/input/serio/i8042-acpipnpio.h @@ -671,6 +671,15 @@ static const struct dmi_system_id i8042_dmi_quirk_table[] __initconst = { }, .driver_data = (void *)(SERIO_QUIRK_NOAUX) }, + { + /* Fujitsu LIFEBOOK U7410 */ + /* Mux mode also disturbs the EC keyboard, causing spurious ACKs */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "FUJITSU CLIENT COMPUTING LIMITED"), + DMI_MATCH(DMI_PRODUCT_NAME, "LIFEBOOK U7410"), + }, + .driver_data = (void *)(SERIO_QUIRK_NOMUX) + }, { /* Gigabyte M912 */ .matches = { diff --git a/drivers/input/touchscreen/s6sy761.c b/drivers/input/touchscreen/s6sy761.c index 0f24a9b73063..899c179fb2d3 100644 --- a/drivers/input/touchscreen/s6sy761.c +++ b/drivers/input/touchscreen/s6sy761.c @@ -500,10 +500,22 @@ static int s6sy761_suspend(struct device *dev) static int s6sy761_resume(struct device *dev) { struct s6sy761_data *sdata = dev_get_drvdata(dev); + int err; + + err = s6sy761_power_on(sdata); + if (err) + return err; enable_irq(sdata->client->irq); - return s6sy761_power_on(sdata); + guard(mutex)(&sdata->input->mutex); + + if (!input_device_enabled(sdata->input)) + return 0; + + err = i2c_smbus_write_byte(sdata->client, S6SY761_SENSE_ON); + + return err; } static const struct dev_pm_ops s6sy761_pm_ops = { diff --git a/drivers/interconnect/qcom/x1e80100.c b/drivers/interconnect/qcom/x1e80100.c index 8075e0ff2059..2ba2823c7860 100644 --- a/drivers/interconnect/qcom/x1e80100.c +++ b/drivers/interconnect/qcom/x1e80100.c @@ -173,13 +173,6 @@ static struct qcom_icc_node qhm_qspi = { .name = "qhm_qspi", .channels = 1, .buswidth = 4, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0xb000 }, - .prio = 2, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_a1noc_snoc }, }; @@ -188,13 +181,6 @@ static struct qcom_icc_node qhm_qup1 = { .name = "qhm_qup1", .channels = 1, .buswidth = 4, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0xc000 }, - .prio = 2, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_a1noc_snoc }, }; @@ -203,13 +189,6 @@ static struct qcom_icc_node xm_sdc4 = { .name = "xm_sdc4", .channels = 1, .buswidth = 8, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0xd000 }, - .prio = 2, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_a1noc_snoc }, }; @@ -218,13 +197,6 @@ static struct qcom_icc_node xm_ufs_mem = { .name = "xm_ufs_mem", .channels = 1, .buswidth = 16, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0xe000 }, - .prio = 2, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_a1noc_snoc }, }; @@ -233,13 +205,6 @@ static struct qcom_icc_node qhm_qup0 = { .name = "qhm_qup0", .channels = 1, .buswidth = 4, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0x16000 }, - .prio = 2, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_a2noc_snoc }, }; @@ -248,13 +213,6 @@ static struct qcom_icc_node qhm_qup2 = { .name = "qhm_qup2", .channels = 1, .buswidth = 4, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0x11000 }, - .prio = 2, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_a2noc_snoc }, }; @@ -263,13 +221,6 @@ static struct qcom_icc_node qxm_crypto = { .name = "qxm_crypto", .channels = 1, .buswidth = 8, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0x12000 }, - .prio = 2, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_a2noc_snoc }, }; @@ -286,13 +237,6 @@ static struct qcom_icc_node xm_qdss_etr_0 = { .name = "xm_qdss_etr_0", .channels = 1, .buswidth = 8, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0x13000 }, - .prio = 2, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_a2noc_snoc }, }; @@ -301,13 +245,6 @@ static struct qcom_icc_node xm_qdss_etr_1 = { .name = "xm_qdss_etr_1", .channels = 1, .buswidth = 8, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0x14000 }, - .prio = 2, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_a2noc_snoc }, }; @@ -316,13 +253,6 @@ static struct qcom_icc_node xm_sdc2 = { .name = "xm_sdc2", .channels = 1, .buswidth = 8, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0x15000 }, - .prio = 2, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_a2noc_snoc }, }; @@ -407,13 +337,6 @@ static struct qcom_icc_node alm_gpu_tcu = { .name = "alm_gpu_tcu", .channels = 1, .buswidth = 8, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0x271000 }, - .prio = 1, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 2, .link_nodes = { &qns_gem_noc_cnoc, &qns_llcc }, }; @@ -422,13 +345,6 @@ static struct qcom_icc_node alm_pcie_tcu = { .name = "alm_pcie_tcu", .channels = 1, .buswidth = 8, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0x27d000 }, - .prio = 3, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 2, .link_nodes = { &qns_gem_noc_cnoc, &qns_llcc }, }; @@ -437,13 +353,6 @@ static struct qcom_icc_node alm_sys_tcu = { .name = "alm_sys_tcu", .channels = 1, .buswidth = 8, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0x273000 }, - .prio = 6, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 2, .link_nodes = { &qns_gem_noc_cnoc, &qns_llcc }, }; @@ -461,13 +370,6 @@ static struct qcom_icc_node qnm_gpu = { .name = "qnm_gpu", .channels = 4, .buswidth = 32, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 4, - .port_offsets = { 0x51000, 0x58000, 0xd1000, 0xd8000 }, - .prio = 0, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 2, .link_nodes = { &qns_gem_noc_cnoc, &qns_llcc }, }; @@ -476,13 +378,6 @@ static struct qcom_icc_node qnm_lpass = { .name = "qnm_lpass", .channels = 1, .buswidth = 16, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0x275000 }, - .prio = 0, - .urg_fwd = 1, - .prio_fwd_disable = 0, - }, .num_links = 3, .link_nodes = { &qns_gem_noc_cnoc, &qns_llcc, &qns_pcie }, @@ -492,13 +387,6 @@ static struct qcom_icc_node qnm_mnoc_hf = { .name = "qnm_mnoc_hf", .channels = 2, .buswidth = 32, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 2, - .port_offsets = { 0x53000, 0xd3000 }, - .prio = 0, - .urg_fwd = 1, - .prio_fwd_disable = 0, - }, .num_links = 2, .link_nodes = { &qns_gem_noc_cnoc, &qns_llcc }, }; @@ -507,13 +395,6 @@ static struct qcom_icc_node qnm_mnoc_sf = { .name = "qnm_mnoc_sf", .channels = 2, .buswidth = 32, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 2, - .port_offsets = { 0x55000, 0xd5000 }, - .prio = 0, - .urg_fwd = 1, - .prio_fwd_disable = 0, - }, .num_links = 2, .link_nodes = { &qns_gem_noc_cnoc, &qns_llcc }, }; @@ -522,13 +403,6 @@ static struct qcom_icc_node qnm_nsp_noc = { .name = "qnm_nsp_noc", .channels = 2, .buswidth = 32, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 2, - .port_offsets = { 0x57000, 0xd7000 }, - .prio = 0, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 3, .link_nodes = { &qns_gem_noc_cnoc, &qns_llcc, &qns_pcie }, @@ -538,13 +412,6 @@ static struct qcom_icc_node qnm_pcie = { .name = "qnm_pcie", .channels = 1, .buswidth = 64, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0x277000 }, - .prio = 0, - .urg_fwd = 1, - .prio_fwd_disable = 0, - }, .num_links = 2, .link_nodes = { &qns_gem_noc_cnoc, &qns_llcc }, }; @@ -553,13 +420,6 @@ static struct qcom_icc_node qnm_snoc_sf = { .name = "qnm_snoc_sf", .channels = 1, .buswidth = 64, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0x27b000 }, - .prio = 2, - .urg_fwd = 1, - .prio_fwd_disable = 0, - }, .num_links = 3, .link_nodes = { &qns_gem_noc_cnoc, &qns_llcc, &qns_pcie }, @@ -569,13 +429,6 @@ static struct qcom_icc_node xm_gic = { .name = "xm_gic", .channels = 1, .buswidth = 8, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0x27f000 }, - .prio = 2, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_llcc }, }; @@ -616,13 +469,6 @@ static struct qcom_icc_node qnm_av1_enc = { .name = "qnm_av1_enc", .channels = 1, .buswidth = 32, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0x2f000 }, - .prio = 4, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_mem_noc_sf }, }; @@ -631,13 +477,6 @@ static struct qcom_icc_node qnm_camnoc_hf = { .name = "qnm_camnoc_hf", .channels = 2, .buswidth = 32, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 2, - .port_offsets = { 0x28000, 0x29000 }, - .prio = 0, - .urg_fwd = 1, - .prio_fwd_disable = 0, - }, .num_links = 1, .link_nodes = { &qns_mem_noc_hf }, }; @@ -646,13 +485,6 @@ static struct qcom_icc_node qnm_camnoc_icp = { .name = "qnm_camnoc_icp", .channels = 1, .buswidth = 8, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0x2a000 }, - .prio = 4, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_mem_noc_sf }, }; @@ -661,13 +493,6 @@ static struct qcom_icc_node qnm_camnoc_sf = { .name = "qnm_camnoc_sf", .channels = 2, .buswidth = 32, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 2, - .port_offsets = { 0x2b000, 0x2c000 }, - .prio = 0, - .urg_fwd = 1, - .prio_fwd_disable = 0, - }, .num_links = 1, .link_nodes = { &qns_mem_noc_sf }, }; @@ -676,13 +501,6 @@ static struct qcom_icc_node qnm_eva = { .name = "qnm_eva", .channels = 1, .buswidth = 32, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0x33000 }, - .prio = 0, - .urg_fwd = 1, - .prio_fwd_disable = 0, - }, .num_links = 1, .link_nodes = { &qns_mem_noc_sf }, }; @@ -691,13 +509,6 @@ static struct qcom_icc_node qnm_mdp = { .name = "qnm_mdp", .channels = 2, .buswidth = 32, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 2, - .port_offsets = { 0x2d000, 0x2e000 }, - .prio = 0, - .urg_fwd = 1, - .prio_fwd_disable = 0, - }, .num_links = 1, .link_nodes = { &qns_mem_noc_hf }, }; @@ -706,13 +517,6 @@ static struct qcom_icc_node qnm_video = { .name = "qnm_video", .channels = 2, .buswidth = 32, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 2, - .port_offsets = { 0x30000, 0x31000 }, - .prio = 0, - .urg_fwd = 1, - .prio_fwd_disable = 0, - }, .num_links = 1, .link_nodes = { &qns_mem_noc_sf }, }; @@ -721,13 +525,6 @@ static struct qcom_icc_node qnm_video_cv_cpu = { .name = "qnm_video_cv_cpu", .channels = 1, .buswidth = 8, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0x32000 }, - .prio = 4, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_mem_noc_sf }, }; @@ -736,13 +533,6 @@ static struct qcom_icc_node qnm_video_v_cpu = { .name = "qnm_video_v_cpu", .channels = 1, .buswidth = 8, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0x34000 }, - .prio = 4, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_mem_noc_sf }, }; @@ -783,13 +573,6 @@ static struct qcom_icc_node xm_pcie_3 = { .name = "xm_pcie_3", .channels = 1, .buswidth = 64, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0x7000 }, - .prio = 2, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_pcie_north_gem_noc }, }; @@ -798,13 +581,6 @@ static struct qcom_icc_node xm_pcie_4 = { .name = "xm_pcie_4", .channels = 1, .buswidth = 8, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0x8000 }, - .prio = 2, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_pcie_north_gem_noc }, }; @@ -813,13 +589,6 @@ static struct qcom_icc_node xm_pcie_5 = { .name = "xm_pcie_5", .channels = 1, .buswidth = 8, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0x9000 }, - .prio = 3, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_pcie_north_gem_noc }, }; @@ -828,13 +597,6 @@ static struct qcom_icc_node xm_pcie_0 = { .name = "xm_pcie_0", .channels = 1, .buswidth = 16, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0x9000 }, - .prio = 2, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_pcie_south_gem_noc }, }; @@ -843,13 +605,6 @@ static struct qcom_icc_node xm_pcie_1 = { .name = "xm_pcie_1", .channels = 1, .buswidth = 16, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0xa000 }, - .prio = 2, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_pcie_south_gem_noc }, }; @@ -858,13 +613,6 @@ static struct qcom_icc_node xm_pcie_2 = { .name = "xm_pcie_2", .channels = 1, .buswidth = 16, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0xb000 }, - .prio = 2, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_pcie_south_gem_noc }, }; @@ -873,13 +621,6 @@ static struct qcom_icc_node xm_pcie_6a = { .name = "xm_pcie_6a", .channels = 1, .buswidth = 32, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0xc000 }, - .prio = 2, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_pcie_south_gem_noc }, }; @@ -888,13 +629,6 @@ static struct qcom_icc_node xm_pcie_6b = { .name = "xm_pcie_6b", .channels = 1, .buswidth = 16, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0xd000 }, - .prio = 2, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_pcie_south_gem_noc }, }; @@ -919,13 +653,6 @@ static struct qcom_icc_node qnm_gic = { .name = "qnm_gic", .channels = 1, .buswidth = 8, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0x1c000 }, - .prio = 2, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_gemnoc_sf }, }; @@ -958,13 +685,6 @@ static struct qcom_icc_node xm_usb2_0 = { .name = "xm_usb2_0", .channels = 1, .buswidth = 8, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0x6000 }, - .prio = 2, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_aggre_usb_north_snoc }, }; @@ -973,13 +693,6 @@ static struct qcom_icc_node xm_usb3_mp = { .name = "xm_usb3_mp", .channels = 1, .buswidth = 16, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0x7000 }, - .prio = 2, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_aggre_usb_north_snoc }, }; @@ -988,13 +701,6 @@ static struct qcom_icc_node xm_usb3_0 = { .name = "xm_usb3_0", .channels = 1, .buswidth = 8, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0xa000 }, - .prio = 2, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_aggre_usb_south_snoc }, }; @@ -1003,13 +709,6 @@ static struct qcom_icc_node xm_usb3_1 = { .name = "xm_usb3_1", .channels = 1, .buswidth = 8, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0xb000 }, - .prio = 2, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_aggre_usb_south_snoc }, }; @@ -1018,13 +717,6 @@ static struct qcom_icc_node xm_usb3_2 = { .name = "xm_usb3_2", .channels = 1, .buswidth = 8, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0xc000 }, - .prio = 2, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_aggre_usb_south_snoc }, }; @@ -1033,13 +725,6 @@ static struct qcom_icc_node xm_usb4_0 = { .name = "xm_usb4_0", .channels = 1, .buswidth = 16, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0xd000 }, - .prio = 2, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_aggre_usb_south_snoc }, }; @@ -1048,13 +733,6 @@ static struct qcom_icc_node xm_usb4_1 = { .name = "xm_usb4_1", .channels = 1, .buswidth = 16, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0xe000 }, - .prio = 2, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_aggre_usb_south_snoc }, }; @@ -1063,13 +741,6 @@ static struct qcom_icc_node xm_usb4_2 = { .name = "xm_usb4_2", .channels = 1, .buswidth = 16, - .qosbox = &(const struct qcom_icc_qosbox) { - .num_ports = 1, - .port_offsets = { 0xf000 }, - .prio = 2, - .urg_fwd = 0, - .prio_fwd_disable = 1, - }, .num_links = 1, .link_nodes = { &qns_aggre_usb_south_snoc }, }; @@ -1795,21 +1466,11 @@ static struct qcom_icc_node * const aggre1_noc_nodes[] = { [SLAVE_A1NOC_SNOC] = &qns_a1noc_snoc, }; -static const struct regmap_config x1e80100_aggre1_noc_regmap_config = { - .reg_bits = 32, - .reg_stride = 4, - .val_bits = 32, - .max_register = 0x14400, - .fast_io = true, -}; - static const struct qcom_icc_desc x1e80100_aggre1_noc = { - .config = &x1e80100_aggre1_noc_regmap_config, .nodes = aggre1_noc_nodes, .num_nodes = ARRAY_SIZE(aggre1_noc_nodes), .bcms = aggre1_noc_bcms, .num_bcms = ARRAY_SIZE(aggre1_noc_bcms), - .qos_requires_clocks = true, }; static struct qcom_icc_bcm * const aggre2_noc_bcms[] = { @@ -1827,16 +1488,7 @@ static struct qcom_icc_node * const aggre2_noc_nodes[] = { [SLAVE_A2NOC_SNOC] = &qns_a2noc_snoc, }; -static const struct regmap_config x1e80100_aggre2_noc_regmap_config = { - .reg_bits = 32, - .reg_stride = 4, - .val_bits = 32, - .max_register = 0x1c400, - .fast_io = true, -}; - static const struct qcom_icc_desc x1e80100_aggre2_noc = { - .config = &x1e80100_aggre2_noc_regmap_config, .nodes = aggre2_noc_nodes, .num_nodes = ARRAY_SIZE(aggre2_noc_nodes), .bcms = aggre2_noc_bcms, @@ -1921,16 +1573,7 @@ static struct qcom_icc_node * const cnoc_cfg_nodes[] = { [SLAVE_TCU] = &xs_sys_tcu_cfg, }; -static const struct regmap_config x1e80100_cnoc_cfg_regmap_config = { - .reg_bits = 32, - .reg_stride = 4, - .val_bits = 32, - .max_register = 0x6600, - .fast_io = true, -}; - static const struct qcom_icc_desc x1e80100_cnoc_cfg = { - .config = &x1e80100_cnoc_cfg_regmap_config, .nodes = cnoc_cfg_nodes, .num_nodes = ARRAY_SIZE(cnoc_cfg_nodes), .bcms = cnoc_cfg_bcms, @@ -1960,16 +1603,7 @@ static struct qcom_icc_node * const cnoc_main_nodes[] = { [SLAVE_PCIE_6B] = &xs_pcie_6b, }; -static const struct regmap_config x1e80100_cnoc_main_regmap_config = { - .reg_bits = 32, - .reg_stride = 4, - .val_bits = 32, - .max_register = 0x14400, - .fast_io = true, -}; - static const struct qcom_icc_desc x1e80100_cnoc_main = { - .config = &x1e80100_cnoc_main_regmap_config, .nodes = cnoc_main_nodes, .num_nodes = ARRAY_SIZE(cnoc_main_nodes), .bcms = cnoc_main_bcms, @@ -1999,16 +1633,7 @@ static struct qcom_icc_node * const gem_noc_nodes[] = { [SLAVE_MEM_NOC_PCIE_SNOC] = &qns_pcie, }; -static const struct regmap_config x1e80100_gem_noc_regmap_config = { - .reg_bits = 32, - .reg_stride = 4, - .val_bits = 32, - .max_register = 0x311200, - .fast_io = true, -}; - static const struct qcom_icc_desc x1e80100_gem_noc = { - .config = &x1e80100_gem_noc_regmap_config, .nodes = gem_noc_nodes, .num_nodes = ARRAY_SIZE(gem_noc_nodes), .bcms = gem_noc_bcms, @@ -2023,16 +1648,7 @@ static struct qcom_icc_node * const lpass_ag_noc_nodes[] = { [SLAVE_LPASS_GEM_NOC] = &qns_lpass_ag_noc_gemnoc, }; -static const struct regmap_config x1e80100_lpass_ag_noc_regmap_config = { - .reg_bits = 32, - .reg_stride = 4, - .val_bits = 32, - .max_register = 0xe080, - .fast_io = true, -}; - static const struct qcom_icc_desc x1e80100_lpass_ag_noc = { - .config = &x1e80100_lpass_ag_noc_regmap_config, .nodes = lpass_ag_noc_nodes, .num_nodes = ARRAY_SIZE(lpass_ag_noc_nodes), .bcms = lpass_ag_noc_bcms, @@ -2048,16 +1664,7 @@ static struct qcom_icc_node * const lpass_lpiaon_noc_nodes[] = { [SLAVE_LPIAON_NOC_LPASS_AG_NOC] = &qns_lpass_aggnoc, }; -static const struct regmap_config x1e80100_lpass_lpiaon_noc_regmap_config = { - .reg_bits = 32, - .reg_stride = 4, - .val_bits = 32, - .max_register = 0x19080, - .fast_io = true, -}; - static const struct qcom_icc_desc x1e80100_lpass_lpiaon_noc = { - .config = &x1e80100_lpass_lpiaon_noc_regmap_config, .nodes = lpass_lpiaon_noc_nodes, .num_nodes = ARRAY_SIZE(lpass_lpiaon_noc_nodes), .bcms = lpass_lpiaon_noc_bcms, @@ -2072,16 +1679,7 @@ static struct qcom_icc_node * const lpass_lpicx_noc_nodes[] = { [SLAVE_LPICX_NOC_LPIAON_NOC] = &qns_lpi_aon_noc, }; -static const struct regmap_config x1e80100_lpass_lpicx_noc_regmap_config = { - .reg_bits = 32, - .reg_stride = 4, - .val_bits = 32, - .max_register = 0x3a200, - .fast_io = true, -}; - static const struct qcom_icc_desc x1e80100_lpass_lpicx_noc = { - .config = &x1e80100_lpass_lpicx_noc_regmap_config, .nodes = lpass_lpicx_noc_nodes, .num_nodes = ARRAY_SIZE(lpass_lpicx_noc_nodes), .bcms = lpass_lpicx_noc_bcms, @@ -2126,16 +1724,7 @@ static struct qcom_icc_node * const mmss_noc_nodes[] = { [SLAVE_SERVICE_MNOC] = &srvc_mnoc, }; -static const struct regmap_config x1e80100_mmss_noc_regmap_config = { - .reg_bits = 32, - .reg_stride = 4, - .val_bits = 32, - .max_register = 0x5b800, - .fast_io = true, -}; - static const struct qcom_icc_desc x1e80100_mmss_noc = { - .config = &x1e80100_mmss_noc_regmap_config, .nodes = mmss_noc_nodes, .num_nodes = ARRAY_SIZE(mmss_noc_nodes), .bcms = mmss_noc_bcms, @@ -2151,16 +1740,7 @@ static struct qcom_icc_node * const nsp_noc_nodes[] = { [SLAVE_CDSP_MEM_NOC] = &qns_nsp_gemnoc, }; -static const struct regmap_config x1e80100_nsp_noc_regmap_config = { - .reg_bits = 32, - .reg_stride = 4, - .val_bits = 32, - .max_register = 0xe080, - .fast_io = true, -}; - static const struct qcom_icc_desc x1e80100_nsp_noc = { - .config = &x1e80100_nsp_noc_regmap_config, .nodes = nsp_noc_nodes, .num_nodes = ARRAY_SIZE(nsp_noc_nodes), .bcms = nsp_noc_bcms, @@ -2177,16 +1757,7 @@ static struct qcom_icc_node * const pcie_center_anoc_nodes[] = { [SLAVE_ANOC_PCIE_GEM_NOC] = &qns_pcie_mem_noc, }; -static const struct regmap_config x1e80100_pcie_center_anoc_regmap_config = { - .reg_bits = 32, - .reg_stride = 4, - .val_bits = 32, - .max_register = 0x7000, - .fast_io = true, -}; - static const struct qcom_icc_desc x1e80100_pcie_center_anoc = { - .config = &x1e80100_pcie_center_anoc_regmap_config, .nodes = pcie_center_anoc_nodes, .num_nodes = ARRAY_SIZE(pcie_center_anoc_nodes), .bcms = pcie_center_anoc_bcms, @@ -2203,16 +1774,7 @@ static struct qcom_icc_node * const pcie_north_anoc_nodes[] = { [SLAVE_PCIE_NORTH] = &qns_pcie_north_gem_noc, }; -static const struct regmap_config x1e80100_pcie_north_anoc_regmap_config = { - .reg_bits = 32, - .reg_stride = 4, - .val_bits = 32, - .max_register = 0x9080, - .fast_io = true, -}; - static const struct qcom_icc_desc x1e80100_pcie_north_anoc = { - .config = &x1e80100_pcie_north_anoc_regmap_config, .nodes = pcie_north_anoc_nodes, .num_nodes = ARRAY_SIZE(pcie_north_anoc_nodes), .bcms = pcie_north_anoc_bcms, @@ -2231,16 +1793,7 @@ static struct qcom_icc_node * const pcie_south_anoc_nodes[] = { [SLAVE_PCIE_SOUTH] = &qns_pcie_south_gem_noc, }; -static const struct regmap_config x1e80100_pcie_south_anoc_regmap_config = { - .reg_bits = 32, - .reg_stride = 4, - .val_bits = 32, - .max_register = 0xd080, - .fast_io = true, -}; - static const struct qcom_icc_desc x1e80100_pcie_south_anoc = { - .config = &x1e80100_pcie_south_anoc_regmap_config, .nodes = pcie_south_anoc_nodes, .num_nodes = ARRAY_SIZE(pcie_south_anoc_nodes), .bcms = pcie_south_anoc_bcms, @@ -2262,16 +1815,7 @@ static struct qcom_icc_node * const system_noc_nodes[] = { [SLAVE_SNOC_GEM_NOC_SF] = &qns_gemnoc_sf, }; -static const struct regmap_config x1e80100_system_noc_regmap_config = { - .reg_bits = 32, - .reg_stride = 4, - .val_bits = 32, - .max_register = 0x1c080, - .fast_io = true, -}; - static const struct qcom_icc_desc x1e80100_system_noc = { - .config = &x1e80100_system_noc_regmap_config, .nodes = system_noc_nodes, .num_nodes = ARRAY_SIZE(system_noc_nodes), .bcms = system_noc_bcms, @@ -2287,16 +1831,7 @@ static struct qcom_icc_node * const usb_center_anoc_nodes[] = { [SLAVE_USB_NOC_SNOC] = &qns_aggre_usb_snoc, }; -static const struct regmap_config x1e80100_usb_center_anoc_regmap_config = { - .reg_bits = 32, - .reg_stride = 4, - .val_bits = 32, - .max_register = 0x8800, - .fast_io = true, -}; - static const struct qcom_icc_desc x1e80100_usb_center_anoc = { - .config = &x1e80100_usb_center_anoc_regmap_config, .nodes = usb_center_anoc_nodes, .num_nodes = ARRAY_SIZE(usb_center_anoc_nodes), .bcms = usb_center_anoc_bcms, @@ -2312,21 +1847,11 @@ static struct qcom_icc_node * const usb_north_anoc_nodes[] = { [SLAVE_AGGRE_USB_NORTH] = &qns_aggre_usb_north_snoc, }; -static const struct regmap_config x1e80100_usb_north_anoc_regmap_config = { - .reg_bits = 32, - .reg_stride = 4, - .val_bits = 32, - .max_register = 0x7080, - .fast_io = true, -}; - static const struct qcom_icc_desc x1e80100_usb_north_anoc = { - .config = &x1e80100_usb_north_anoc_regmap_config, .nodes = usb_north_anoc_nodes, .num_nodes = ARRAY_SIZE(usb_north_anoc_nodes), .bcms = usb_north_anoc_bcms, .num_bcms = ARRAY_SIZE(usb_north_anoc_bcms), - .qos_requires_clocks = true, }; static struct qcom_icc_bcm * const usb_south_anoc_bcms[] = { @@ -2342,21 +1867,11 @@ static struct qcom_icc_node * const usb_south_anoc_nodes[] = { [SLAVE_AGGRE_USB_SOUTH] = &qns_aggre_usb_south_snoc, }; -static const struct regmap_config x1e80100_usb_south_anoc_regmap_config = { - .reg_bits = 32, - .reg_stride = 4, - .val_bits = 32, - .max_register = 0xf080, - .fast_io = true, -}; - static const struct qcom_icc_desc x1e80100_usb_south_anoc = { - .config = &x1e80100_usb_south_anoc_regmap_config, .nodes = usb_south_anoc_nodes, .num_nodes = ARRAY_SIZE(usb_south_anoc_nodes), .bcms = usb_south_anoc_bcms, .num_bcms = ARRAY_SIZE(usb_south_anoc_bcms), - .qos_requires_clocks = true, }; static const struct of_device_id qnoc_of_match[] = { diff --git a/drivers/mtd/chips/cfi_cmdset_0001.c b/drivers/mtd/chips/cfi_cmdset_0001.c index 6049ba2d6bcb..60d82a1acb01 100644 --- a/drivers/mtd/chips/cfi_cmdset_0001.c +++ b/drivers/mtd/chips/cfi_cmdset_0001.c @@ -1716,12 +1716,24 @@ static int cfi_intelext_write_words (struct mtd_info *mtd, loff_t to , size_t le } +/* + * Keep noinline: inlined, the map_word temporaries put do_write_buffer() over + * the frame-size limit with MTD_MAP_BANK_WIDTH_32 and KASAN_STACK. + */ +static noinline void __xipram cfi_write_cmd(struct map_info *map, + unsigned long cmd, unsigned long adr) +{ + struct cfi_private *cfi = map->fldrv_priv; + + map_write(map, CMD(cmd), adr); +} + static int __xipram do_write_buffer(struct map_info *map, struct flchip *chip, unsigned long adr, const struct kvec **pvec, unsigned long *pvec_seek, int len) { struct cfi_private *cfi = map->fldrv_priv; - map_word status, write_cmd, datum; + map_word status, datum; unsigned long cmd_adr; int ret, wbufsize, word_gap, words; const struct kvec *vec; @@ -1740,9 +1752,6 @@ static int __xipram do_write_buffer(struct map_info *map, struct flchip *chip, if (is_LH28F640BF(cfi)) cmd_adr = adr; - /* Let's determine this according to the interleave only once */ - write_cmd = (cfi->cfiq->P_ID != P_ID_INTEL_PERFORMANCE) ? CMD(0xe8) : CMD(0xe9); - mutex_lock(&chip->mutex); ret = get_chip(map, chip, cmd_adr, FL_WRITING); if (ret) { @@ -1759,7 +1768,7 @@ static int __xipram do_write_buffer(struct map_info *map, struct flchip *chip, So we must check here and reset those bits if they're set. Otherwise we're just pissing in the wind */ if (chip->state != FL_STATUS) { - map_write(map, CMD(0x70), cmd_adr); + cfi_write_cmd(map, 0x70, cmd_adr); chip->state = FL_STATUS; } status = map_read(map, cmd_adr); @@ -1767,21 +1776,22 @@ static int __xipram do_write_buffer(struct map_info *map, struct flchip *chip, xip_enable(map, chip, cmd_adr); printk(KERN_WARNING "SR.4 or SR.5 bits set in buffer write (status %lx). Clearing.\n", status.x[0]); xip_disable(map, chip, cmd_adr); - map_write(map, CMD(0x50), cmd_adr); - map_write(map, CMD(0x70), cmd_adr); + cfi_write_cmd(map, 0x50, cmd_adr); + cfi_write_cmd(map, 0x70, cmd_adr); } chip->state = FL_WRITING_TO_BUFFER; - map_write(map, write_cmd, cmd_adr); + cfi_write_cmd(map, (cfi->cfiq->P_ID != P_ID_INTEL_PERFORMANCE) ? 0xe8 : 0xe9, + cmd_adr); ret = WAIT_TIMEOUT(map, chip, cmd_adr, 0, 0); if (ret) { /* Argh. Not ready for write to buffer */ map_word Xstatus = map_read(map, cmd_adr); - map_write(map, CMD(0x70), cmd_adr); + cfi_write_cmd(map, 0x70, cmd_adr); chip->state = FL_STATUS; status = map_read(map, cmd_adr); - map_write(map, CMD(0x50), cmd_adr); - map_write(map, CMD(0x70), cmd_adr); + cfi_write_cmd(map, 0x50, cmd_adr); + cfi_write_cmd(map, 0x70, cmd_adr); xip_enable(map, chip, cmd_adr); printk(KERN_ERR "%s: Chip not ready for buffer write. Xstatus = %lx, status = %lx\n", map->name, Xstatus.x[0], status.x[0]); @@ -1800,7 +1810,7 @@ static int __xipram do_write_buffer(struct map_info *map, struct flchip *chip, } /* Write length of data to come */ - map_write(map, CMD(words), cmd_adr ); + cfi_write_cmd(map, words, cmd_adr); /* Write data */ vec = *pvec; @@ -1837,7 +1847,7 @@ static int __xipram do_write_buffer(struct map_info *map, struct flchip *chip, *pvec_seek = vec_seek; /* GO GO GO */ - map_write(map, CMD(0xd0), cmd_adr); + cfi_write_cmd(map, 0xd0, cmd_adr); chip->state = FL_WRITING; ret = INVAL_CACHE_AND_WAIT(map, chip, cmd_adr, @@ -1845,7 +1855,7 @@ static int __xipram do_write_buffer(struct map_info *map, struct flchip *chip, chip->buffer_write_time, chip->buffer_write_time_max); if (ret) { - map_write(map, CMD(0x70), cmd_adr); + cfi_write_cmd(map, 0x70, cmd_adr); chip->state = FL_STATUS; xip_enable(map, chip, cmd_adr); printk(KERN_ERR "%s: buffer write error (status timeout)\n", map->name); @@ -1858,8 +1868,8 @@ static int __xipram do_write_buffer(struct map_info *map, struct flchip *chip, unsigned long chipstatus = MERGESTATUS(status); /* reset status */ - map_write(map, CMD(0x50), cmd_adr); - map_write(map, CMD(0x70), cmd_adr); + cfi_write_cmd(map, 0x50, cmd_adr); + cfi_write_cmd(map, 0x70, cmd_adr); xip_enable(map, chip, cmd_adr); if (chipstatus & 0x02) { diff --git a/drivers/mtd/devices/block2mtd.c b/drivers/mtd/devices/block2mtd.c index 03e80b2c4f5a..349fa07be314 100644 --- a/drivers/mtd/devices/block2mtd.c +++ b/drivers/mtd/devices/block2mtd.c @@ -293,7 +293,7 @@ static struct block2mtd_dev *add_device(char *devname, int erase_size, } size = bdev_nr_bytes(bdev); - if ((long)size % erase_size) { + if (!erase_size || (long)size % erase_size) { pr_err("erasesize must be a divisor of device size\n"); goto err_free_block2mtd; } diff --git a/drivers/mtd/devices/mtd_intel_dg.c b/drivers/mtd/devices/mtd_intel_dg.c index a566e86eb5e3..55a945c1a67a 100644 --- a/drivers/mtd/devices/mtd_intel_dg.c +++ b/drivers/mtd/devices/mtd_intel_dg.c @@ -387,7 +387,7 @@ idg_erase(struct intel_dg_nvm *nvm, u8 region, loff_t from, u64 len, u64 *fail_a void __iomem *base2 = nvm->base2; void __iomem *base = nvm->base; const u32 block = 0x10; - u32 iter = 0; + u32 iter; u32 reg; u64 i; @@ -396,6 +396,7 @@ idg_erase(struct intel_dg_nvm *nvm, u8 region, loff_t from, u64 len, u64 *fail_a iowrite32(region << 24 | block, base + NVM_ERASE_REG); if (nvm->non_posted_erase) { /* Wait for Erase Done */ + iter = 0; reg = ioread32(base2 + NVM_DEBUG_REG); while (!(reg & NVM_NON_POSTED_ERASE_DONE) && ++iter < NVM_NON_POSTED_ERASE_DONE_ITER) { diff --git a/drivers/mtd/mtdcore.c b/drivers/mtd/mtdcore.c index 16629382a787..491a27d0315f 100644 --- a/drivers/mtd/mtdcore.c +++ b/drivers/mtd/mtdcore.c @@ -1083,6 +1083,7 @@ static int mtd_otp_nvmem_add(struct mtd_info *mtd) err: nvmem_unregister(mtd->otp_user_nvmem); + mtd->otp_user_nvmem = NULL; /* Don't report error if OTP is not supported. */ if (err == -EOPNOTSUPP) return 0; @@ -1332,7 +1333,7 @@ int __get_mtd_device(struct mtd_info *mtd) int err; if (master->_get_device) { - err = master->_get_device(mtd); + err = master->_get_device(master); if (err) return err; } diff --git a/drivers/mtd/nand/ecc.c b/drivers/mtd/nand/ecc.c index 6ccdff3fc913..1d809cbd47e6 100644 --- a/drivers/mtd/nand/ecc.c +++ b/drivers/mtd/nand/ecc.c @@ -159,6 +159,13 @@ int nand_ecc_finish_io_req(struct nand_device *nand, } EXPORT_SYMBOL(nand_ecc_finish_io_req); +bool nand_ecc_is_pipelined(const struct nand_device *nand) +{ + return nand->ecc.engine && + nand->ecc.engine->integration == NAND_ECC_ENGINE_INTEGRATION_PIPELINED; +} +EXPORT_SYMBOL(nand_ecc_is_pipelined); + /* Define default OOB placement schemes for large and small page devices */ static int nand_ooblayout_ecc_sp(struct mtd_info *mtd, int section, struct mtd_oob_region *oobregion) diff --git a/drivers/mtd/nand/raw/cadence-nand-controller.c b/drivers/mtd/nand/raw/cadence-nand-controller.c index d53b35a8b3cb..2085632631c0 100644 --- a/drivers/mtd/nand/raw/cadence-nand-controller.c +++ b/drivers/mtd/nand/raw/cadence-nand-controller.c @@ -3143,6 +3143,9 @@ static int cadence_nand_init(struct cdns_nand_ctrl *cdns_ctrl) goto free_buf_desc; } + spin_lock_init(&cdns_ctrl->irq_lock); + init_completion(&cdns_ctrl->complete); + if (devm_request_irq(cdns_ctrl->dev, cdns_ctrl->irq, cadence_nand_isr, IRQF_SHARED, "cadence-nand-controller", cdns_ctrl)) { @@ -3151,9 +3154,6 @@ static int cadence_nand_init(struct cdns_nand_ctrl *cdns_ctrl) goto free_buf; } - spin_lock_init(&cdns_ctrl->irq_lock); - init_completion(&cdns_ctrl->complete); - ret = cadence_nand_hw_init(cdns_ctrl); if (ret) goto disable_irq; diff --git a/drivers/mtd/nand/raw/vf610_nfc.c b/drivers/mtd/nand/raw/vf610_nfc.c index 9940681810cf..1b9b370adfab 100644 --- a/drivers/mtd/nand/raw/vf610_nfc.c +++ b/drivers/mtd/nand/raw/vf610_nfc.c @@ -505,6 +505,11 @@ static int vf610_nfc_exec_op(struct nand_chip *chip, check_only); } +static unsigned int vf610_nfc_spare_size(struct mtd_info *mtd) +{ + return min_t(unsigned int, mtd->oobsize, 64); +} + static inline int vf610_nfc_correct_data(struct nand_chip *chip, uint8_t *dat, uint8_t *oob, int page) { @@ -514,6 +519,7 @@ static inline int vf610_nfc_correct_data(struct nand_chip *chip, uint8_t *dat, u8 ecc_status; u8 ecc_count; int flips_threshold = nfc->chip.ecc.strength / 2; + int ret; ecc_status = vf610_nfc_read(nfc, ecc_status_off) & 0xff; ecc_count = ecc_status & ECC_STATUS_ERR_COUNT; @@ -521,16 +527,22 @@ static inline int vf610_nfc_correct_data(struct nand_chip *chip, uint8_t *dat, if (!(ecc_status & ECC_STATUS_MASK)) return ecc_count; + /* The failed decode leaves a bogus correction in SRAM; re-read without ECC */ nfc->data_access = true; - nand_read_oob_op(&nfc->chip, page, 0, oob, mtd->oobsize); + ret = nand_read_page_op(&nfc->chip, page, 0, dat, mtd->writesize); + if (!ret) + ret = nand_read_oob_op(&nfc->chip, page, 0, oob, + vf610_nfc_spare_size(mtd)); nfc->data_access = false; + if (ret) + return ret; /* - * On an erased page, bit count (including OOB) should be zero or - * at least less then half of the ECC strength. + * Run the erased-page check with the driver's historic threshold + * of half the ECC strength. */ - return nand_check_erased_ecc_chunk(dat, nfc->chip.ecc.size, oob, - mtd->oobsize, NULL, 0, + return nand_check_erased_ecc_chunk(dat, mtd->writesize, oob, + vf610_nfc_spare_size(mtd), NULL, 0, flips_threshold); } @@ -551,7 +563,7 @@ static int vf610_nfc_read_page(struct nand_chip *chip, uint8_t *buf, { struct vf610_nfc *nfc = chip_to_nfc(chip); struct mtd_info *mtd = nand_to_mtd(chip); - int trfr_sz = mtd->writesize + mtd->oobsize; + int trfr_sz = mtd->writesize + vf610_nfc_spare_size(mtd); u32 row = 0, cmd1 = 0, cmd2 = 0, code = 0; int stat; @@ -577,11 +589,16 @@ static int vf610_nfc_read_page(struct nand_chip *chip, uint8_t *buf, */ vf610_nfc_rd_from_sram(buf, nfc->regs + NFC_MAIN_AREA(0), mtd->writesize, false); - if (oob_required) + if (oob_required) { + unsigned int spare = vf610_nfc_spare_size(mtd); + vf610_nfc_rd_from_sram(chip->oob_poi, nfc->regs + NFC_MAIN_AREA(0) + mtd->writesize, - mtd->oobsize, false); + spare, false); + /* Not transferred, and never written: reads back erased */ + memset(chip->oob_poi + spare, 0xff, mtd->oobsize - spare); + } stat = vf610_nfc_correct_data(chip, buf, chip->oob_poi, page); @@ -599,7 +616,7 @@ static int vf610_nfc_write_page(struct nand_chip *chip, const uint8_t *buf, { struct vf610_nfc *nfc = chip_to_nfc(chip); struct mtd_info *mtd = nand_to_mtd(chip); - int trfr_sz = mtd->writesize + mtd->oobsize; + int trfr_sz = mtd->writesize + vf610_nfc_spare_size(mtd); u32 row = 0, cmd1 = 0, cmd2 = 0, code = 0; u8 status; int ret; @@ -740,6 +757,49 @@ static void vf610_nfc_init_controller(struct vf610_nfc *nfc) } } +/* + * With 64 byte OOB chips the core's large page layout matches what + * U-Boot uses, and on chips with more, U-Boot and older kernels clamped + * mtd->oobsize to 64. Modifying the OOB size is no longer possible, the + * actual chip geometry must be respected, so to avoid breaking those + * existing setups use our own layout: the core's large page one, + * computed over the first 64 spare bytes only. + */ +static int vf610_nfc_ooblayout_ecc(struct mtd_info *mtd, int section, + struct mtd_oob_region *oobregion) +{ + struct nand_device *nand = mtd_to_nanddev(mtd); + unsigned int total_ecc_bytes = nand->ecc.ctx.total; + + if (section || !total_ecc_bytes) + return -ERANGE; + + oobregion->length = total_ecc_bytes; + oobregion->offset = vf610_nfc_spare_size(mtd) - oobregion->length; + + return 0; +} + +static int vf610_nfc_ooblayout_free(struct mtd_info *mtd, int section, + struct mtd_oob_region *oobregion) +{ + struct nand_device *nand = mtd_to_nanddev(mtd); + unsigned int total_ecc_bytes = nand->ecc.ctx.total; + + if (section) + return -ERANGE; + + oobregion->length = vf610_nfc_spare_size(mtd) - total_ecc_bytes - 2; + oobregion->offset = 2; + + return 0; +} + +static const struct mtd_ooblayout_ops vf610_nfc_ooblayout_ops = { + .ecc = vf610_nfc_ooblayout_ecc, + .free = vf610_nfc_ooblayout_free, +}; + static int vf610_nfc_attach_chip(struct nand_chip *chip) { struct mtd_info *mtd = nand_to_mtd(chip); @@ -770,12 +830,7 @@ static int vf610_nfc_attach_chip(struct nand_chip *chip) return -ENXIO; } - /* Only 64 byte ECC layouts known */ - if (mtd->oobsize > 64) - mtd->oobsize = 64; - - /* Use default large page ECC layout defined in NAND core */ - mtd_set_ooblayout(mtd, nand_get_large_page_ooblayout()); + mtd_set_ooblayout(mtd, &vf610_nfc_ooblayout_ops); if (chip->ecc.strength == 32) { nfc->ecc_mode = ECC_60_BYTE; chip->ecc.bytes = 60; diff --git a/drivers/mtd/nand/spi/core.c b/drivers/mtd/nand/spi/core.c index 8bf9301f25e7..95353777d7f7 100644 --- a/drivers/mtd/nand/spi/core.c +++ b/drivers/mtd/nand/spi/core.c @@ -281,8 +281,26 @@ static int spinand_init_cfg_cache(struct spinand_device *spinand) static int spinand_init_quad_enable(struct spinand_device *spinand, bool enable) { - return spinand_upd_cfg(spinand, CFG_QUAD_ENABLE, - enable ? CFG_QUAD_ENABLE : 0); + struct nand_device *nand = spinand_to_nand(spinand); + unsigned int target; + int ret; + + /* + * QE is a per-die setting on some devices. Program each target + * individually when enabling or disabling quad I/O mode. + */ + for (target = 0; target < nand->memorg.ntargets; target++) { + ret = spinand_select_target(spinand, target); + if (ret) + return ret; + + ret = spinand_upd_cfg(spinand, CFG_QUAD_ENABLE, + enable ? CFG_QUAD_ENABLE : 0); + if (ret) + return ret; + } + + return 0; } static int spinand_ecc_enable(struct spinand_device *spinand, @@ -508,8 +526,7 @@ static int spinand_read_from_cache_op(struct spinand_device *spinand, else rdesc->info.op_tmpl = &rdesc->info.primary_op_tmpl; - if (nand->ecc.engine->integration == NAND_ECC_ENGINE_INTEGRATION_PIPELINED && - req->mode != MTD_OPS_RAW) + if (nand_ecc_is_pipelined(nand) && req->mode != MTD_OPS_RAW) rdesc->info.op_tmpl->data.ecc = true; else rdesc->info.op_tmpl->data.ecc = false; @@ -603,8 +620,7 @@ static int spinand_write_to_cache_op(struct spinand_device *spinand, wdesc = spinand->dirmaps[req->pos.plane].wdesc; - if (nand->ecc.engine->integration == NAND_ECC_ENGINE_INTEGRATION_PIPELINED && - req->mode != MTD_OPS_RAW) + if (nand_ecc_is_pipelined(nand) && req->mode != MTD_OPS_RAW) wdesc->info.op_tmpl->data.ecc = true; else wdesc->info.op_tmpl->data.ecc = false; @@ -1261,7 +1277,7 @@ static int spinand_create_dirmap(struct spinand_device *spinand, struct spi_mem_dirmap_desc *desc; bool enable_ecc = false, secondary_op = false; - if (nand->ecc.engine->integration == NAND_ECC_ENGINE_INTEGRATION_PIPELINED) + if (nand_ecc_is_pipelined(nand)) enable_ecc = true; if (spinand->cont_read_possible && spinand->op_templates->cont_read_cache) @@ -1797,11 +1813,11 @@ static int spinand_configure_chip(struct spinand_device *spinand) spinand->ssdr_op_templates.write_cache->data.buswidth == 4 || spinand->ssdr_op_templates.update_cache->data.buswidth == 4) quad_enable = true; - } - ret = spinand_init_quad_enable(spinand, quad_enable); - if (ret) - return ret; + ret = spinand_init_quad_enable(spinand, quad_enable); + if (ret) + return ret; + } if (spinand->configure_chip) { ret = spinand->configure_chip(spinand, SSDR); @@ -1809,7 +1825,7 @@ static int spinand_configure_chip(struct spinand_device *spinand) return ret; } - return ret; + return 0; } static int spinand_init_flash(struct spinand_device *spinand) @@ -1980,12 +1996,17 @@ static int spinand_init(struct spinand_device *spinand) goto err_cleanup_ecc_engine; } - if (nand->ecc.engine) { - ret = mtd_ooblayout_count_freebytes(mtd); - if (ret < 0) - goto err_cleanup_ecc_engine; + if (!nand->ecc.engine) { + if (spinand->eccinfo.ooblayout) + mtd_set_ooblayout(mtd, spinand->eccinfo.ooblayout); + else + mtd_set_ooblayout(mtd, &spinand_noecc_ooblayout); } + ret = mtd_ooblayout_count_freebytes(mtd); + if (ret < 0) + goto err_cleanup_ecc_engine; + mtd->oobavail = ret; /* Propagate ECC information to mtd_info */ diff --git a/drivers/mtd/spi-nor/core.c b/drivers/mtd/spi-nor/core.c index ccf4396cdcd0..8bc117b46e02 100644 --- a/drivers/mtd/spi-nor/core.c +++ b/drivers/mtd/spi-nor/core.c @@ -1310,7 +1310,7 @@ static bool spi_nor_rww_start_exclusive(struct spi_nor *nor) { struct spi_nor_rww *rww = &nor->rww; - mutex_lock(&nor->lock); + guard(mutex)(&nor->lock); if (rww->ongoing_io || rww->ongoing_rd || rww->ongoing_pe) return false; diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index a9bff7663eec..de2489c3d9bf 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -490,7 +490,7 @@ static int bond_ipsec_add_sa(struct net_device *bond_dev, !real_dev->xfrmdev_ops->xdo_dev_state_add || netif_is_bond_master(real_dev)) { NL_SET_ERR_MSG_MOD(extack, "Slave does not support ipsec offload"); - err = -EINVAL; + err = -EOPNOTSUPP; goto out; } diff --git a/drivers/net/dsa/mt7530-mdio.c b/drivers/net/dsa/mt7530-mdio.c index 784dd58a7158..de42f70afcfa 100644 --- a/drivers/net/dsa/mt7530-mdio.c +++ b/drivers/net/dsa/mt7530-mdio.c @@ -227,15 +227,17 @@ mt7530_remove(struct mdio_device *mdiodev) if (!priv) return; - ret = regulator_disable(priv->core_pwr); - if (ret < 0) - dev_err(priv->dev, - "Failed to disable core power: %d\n", ret); + if (priv->id == ID_MT7530) { + ret = regulator_disable(priv->core_pwr); + if (ret < 0) + dev_err(priv->dev, + "Failed to disable core power: %d\n", ret); - ret = regulator_disable(priv->io_pwr); - if (ret < 0) - dev_err(priv->dev, "Failed to disable io pwr: %d\n", - ret); + ret = regulator_disable(priv->io_pwr); + if (ret < 0) + dev_err(priv->dev, "Failed to disable io pwr: %d\n", + ret); + } mt7530_remove_common(priv); diff --git a/drivers/net/dsa/mt7530.c b/drivers/net/dsa/mt7530.c index 3e61eb3c2b1e..96832852c65a 100644 --- a/drivers/net/dsa/mt7530.c +++ b/drivers/net/dsa/mt7530.c @@ -3593,9 +3593,6 @@ EXPORT_SYMBOL_GPL(mt7530_probe_common); void mt7530_remove_common(struct mt7530_priv *priv) { - if (priv->irq_domain) - mt7530_free_mdio_irq(priv); - dsa_unregister_switch(priv->ds); mutex_destroy(&priv->reg_mutex); diff --git a/drivers/net/dsa/mv88e6xxx/chip.c b/drivers/net/dsa/mv88e6xxx/chip.c index 7f68a0c55802..a4a8c7e11bf4 100644 --- a/drivers/net/dsa/mv88e6xxx/chip.c +++ b/drivers/net/dsa/mv88e6xxx/chip.c @@ -5639,6 +5639,68 @@ static const struct mv88e6xxx_ops mv88e6390x_ops = { .pcs_ops = &mv88e6390_pcs_ops, }; +static const struct mv88e6xxx_ops mv88e6191x_ops = { + /* MV88E6XXX_FAMILY_6393 without AVB and PTP: 6191X and 6193X */ + .irl_init_all = mv88e6390_g2_irl_init_all, + .get_eeprom = mv88e6xxx_g2_get_eeprom8, + .set_eeprom = mv88e6xxx_g2_set_eeprom8, + .set_switch_mac = mv88e6xxx_g2_set_switch_mac, + .phy_read = mv88e6xxx_g2_smi_phy_read_c22, + .phy_write = mv88e6xxx_g2_smi_phy_write_c22, + .phy_read_c45 = mv88e6xxx_g2_smi_phy_read_c45, + .phy_write_c45 = mv88e6xxx_g2_smi_phy_write_c45, + .port_set_link = mv88e6xxx_port_set_link, + .port_sync_link = mv88e6xxx_port_sync_link, + .port_set_rgmii_delay = mv88e6390_port_set_rgmii_delay, + .port_set_speed_duplex = mv88e6393x_port_set_speed_duplex, + .port_tag_remap = mv88e6390_port_tag_remap, + .port_set_policy = mv88e6393x_port_set_policy, + .port_set_frame_mode = mv88e6351_port_set_frame_mode, + .port_set_ucast_flood = mv88e6352_port_set_ucast_flood, + .port_set_mcast_flood = mv88e6352_port_set_mcast_flood, + .port_set_ether_type = mv88e6393x_port_set_ether_type, + .port_set_jumbo_size = mv88e6165_port_set_jumbo_size, + .port_egress_rate_limiting = mv88e6097_port_egress_rate_limiting, + .port_pause_limit = mv88e6390_port_pause_limit, + .port_disable_learn_limit = mv88e6xxx_port_disable_learn_limit, + .port_disable_pri_override = mv88e6xxx_port_disable_pri_override, + .port_get_cmode = mv88e6352_port_get_cmode, + .port_set_cmode = mv88e6393x_port_set_cmode, + .port_setup_message_port = mv88e6xxx_setup_message_port, + .port_set_upstream_port = mv88e6393x_port_set_upstream_port, + .port_enable_tcam = mv88e6xxx_port_enable_tcam, + .stats_snapshot = mv88e6390_g1_stats_snapshot, + .stats_set_histogram = mv88e6390_g1_stats_set_histogram, + .stats_get_sset_count = mv88e6320_stats_get_sset_count, + .stats_get_strings = mv88e6320_stats_get_strings, + .stats_get_stat = mv88e6390_stats_get_stat, + /* .set_cpu_port is missing because this family does not support a global + * CPU port, only per port CPU port which is set via + * .port_set_upstream_port method. + */ + .set_egress_port = mv88e6393x_set_egress_port, + .watchdog_ops = &mv88e6393x_watchdog_ops, + .mgmt_rsvd2cpu = mv88e6393x_port_mgmt_rsvd2cpu, + .pot_clear = mv88e6xxx_g2_pot_clear, + .hardware_reset_pre = mv88e6xxx_g2_eeprom_wait, + .hardware_reset_post = mv88e6xxx_g2_eeprom_wait, + .reset = mv88e6352_g1_reset, + .rmu_disable = mv88e6390_g1_rmu_disable, + .atu_get_hash = mv88e6165_g1_atu_get_hash, + .atu_set_hash = mv88e6165_g1_atu_set_hash, + .vtu_getnext = mv88e6390_g1_vtu_getnext, + .vtu_loadpurge = mv88e6390_g1_vtu_loadpurge, + .stu_getnext = mv88e6390_g1_stu_getnext, + .stu_loadpurge = mv88e6390_g1_stu_loadpurge, + .serdes_get_lane = mv88e6393x_serdes_get_lane, + .serdes_irq_mapping = mv88e6390_serdes_irq_mapping, + /* TODO: serdes stats */ + .gpio_ops = &mv88e6352_gpio_ops, + .phylink_get_caps = mv88e6393x_phylink_get_caps, + .pcs_ops = &mv88e6393x_pcs_ops, + .tcam_ops = &mv88e6393_tcam_ops, +}; + static const struct mv88e6xxx_ops mv88e6393x_ops = { /* MV88E6XXX_FAMILY_6393 */ .irl_init_all = mv88e6390_g2_irl_init_all, @@ -6163,8 +6225,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = { .atu_move_port_mask = 0x1f, .pvt = true, .multi_chip = true, - .ptp_support = true, - .ops = &mv88e6393x_ops, + .ops = &mv88e6191x_ops, }, [MV88E6193X] = { @@ -6190,8 +6251,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = { .atu_move_port_mask = 0x1f, .pvt = true, .multi_chip = true, - .ptp_support = true, - .ops = &mv88e6393x_ops, + .ops = &mv88e6191x_ops, }, [MV88E6220] = { diff --git a/drivers/net/ethernet/airoha/airoha_npu.c b/drivers/net/ethernet/airoha/airoha_npu.c index 5bb4817a898d..4d3195eb00f7 100644 --- a/drivers/net/ethernet/airoha/airoha_npu.c +++ b/drivers/net/ethernet/airoha/airoha_npu.c @@ -5,6 +5,7 @@ */ #include +#include #include #include #include @@ -751,12 +752,15 @@ static int airoha_npu_probe(struct platform_device *pdev) if (irq < 0) return irq; + err = devm_work_autocancel(dev, &core->wdt_work, + airoha_npu_wdt_work); + if (err) + return err; + err = devm_request_irq(dev, irq, airoha_npu_wdt_handler, IRQF_SHARED, "airoha-npu-wdt", core); if (err) return err; - - INIT_WORK(&core->wdt_work, airoha_npu_wdt_work); } /* wlan IRQ lines */ @@ -803,18 +807,8 @@ static int airoha_npu_probe(struct platform_device *pdev) return 0; } -static void airoha_npu_remove(struct platform_device *pdev) -{ - struct airoha_npu *npu = platform_get_drvdata(pdev); - int i; - - for (i = 0; i < ARRAY_SIZE(npu->cores); i++) - cancel_work_sync(&npu->cores[i].wdt_work); -} - static struct platform_driver airoha_npu_driver = { .probe = airoha_npu_probe, - .remove = airoha_npu_remove, .driver = { .name = "airoha-npu", .of_match_table = of_airoha_npu_match, diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c index ea89619039d8..7eb6456ed0d5 100644 --- a/drivers/net/ethernet/amazon/ena/ena_netdev.c +++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c @@ -4122,6 +4122,8 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent) err_device_destroy: ena_com_delete_host_info(ena_dev); ena_com_admin_destroy(ena_dev); + ena_phc_destroy(adapter); + ena_com_mmio_reg_read_request_destroy(ena_dev); ena_devlink_destroy: ena_devlink_free(devlink); err_metrics_destroy: diff --git a/drivers/net/ethernet/atheros/atl1c/atl1c_main.c b/drivers/net/ethernet/atheros/atl1c/atl1c_main.c index 7efa3fc257b3..e58f1d2c26bd 100644 --- a/drivers/net/ethernet/atheros/atl1c/atl1c_main.c +++ b/drivers/net/ethernet/atheros/atl1c/atl1c_main.c @@ -1602,6 +1602,9 @@ static int atl1c_clean_tx(struct napi_struct *napi, int budget) AT_READ_REGW(&adapter->hw, atl1c_qregs[tpd_ring->num].tpd_cons, &hw_next_to_clean); + if (unlikely(hw_next_to_clean >= tpd_ring->count)) + hw_next_to_clean = next_to_clean; + while (next_to_clean != hw_next_to_clean) { buffer_info = &tpd_ring->buffer_info[next_to_clean]; if (buffer_info->skb) { diff --git a/drivers/net/ethernet/atheros/atl1e/atl1e_main.c b/drivers/net/ethernet/atheros/atl1e/atl1e_main.c index 40290028580b..437989edb780 100644 --- a/drivers/net/ethernet/atheros/atl1e/atl1e_main.c +++ b/drivers/net/ethernet/atheros/atl1e/atl1e_main.c @@ -1234,6 +1234,9 @@ static bool atl1e_clean_tx_irq(struct atl1e_adapter *adapter) u16 hw_next_to_clean = AT_READ_REGW(&adapter->hw, REG_TPD_CONS_IDX); u16 next_to_clean = atomic_read(&tx_ring->next_to_clean); + if (unlikely(hw_next_to_clean >= tx_ring->count)) + hw_next_to_clean = next_to_clean; + while (next_to_clean != hw_next_to_clean) { tx_buffer = &tx_ring->tx_buffer[next_to_clean]; if (tx_buffer->dma) { diff --git a/drivers/net/ethernet/atheros/atlx/atl1.c b/drivers/net/ethernet/atheros/atlx/atl1.c index 98a4d089270e..957d5598dda5 100644 --- a/drivers/net/ethernet/atheros/atlx/atl1.c +++ b/drivers/net/ethernet/atheros/atlx/atl1.c @@ -2066,6 +2066,9 @@ static int atl1_intr_tx(struct atl1_adapter *adapter) sw_tpd_next_to_clean = atomic_read(&tpd_ring->next_to_clean); cmb_tpd_next_to_clean = le16_to_cpu(adapter->cmb.cmb->tpd_cons_idx); + if (unlikely(cmb_tpd_next_to_clean >= tpd_ring->count)) + cmb_tpd_next_to_clean = sw_tpd_next_to_clean; + while (cmb_tpd_next_to_clean != sw_tpd_next_to_clean) { buffer_info = &tpd_ring->buffer_info[sw_tpd_next_to_clean]; if (buffer_info->dma) { diff --git a/drivers/net/ethernet/broadcom/bcmsysport.c b/drivers/net/ethernet/broadcom/bcmsysport.c index 4d06c6ba6641..78909f516823 100644 --- a/drivers/net/ethernet/broadcom/bcmsysport.c +++ b/drivers/net/ethernet/broadcom/bcmsysport.c @@ -391,8 +391,10 @@ static void bcm_sysport_update_mib_counters(struct bcm_sysport_priv *priv) if (priv->is_lite) continue; - if (s->type != BCM_SYSPORT_STAT_MIB_RX) + if (s->type == BCM_SYSPORT_STAT_MIB_TX) offset = UMAC_MIB_STAT_OFFSET; + else if (s->type == BCM_SYSPORT_STAT_RUNT) + offset = 2 * UMAC_MIB_STAT_OFFSET; val = umac_readl(priv, UMAC_MIB_START + j + offset); break; case BCM_SYSPORT_STAT_RXCHK: @@ -482,10 +484,10 @@ static void bcm_sysport_get_stats(struct net_device *dev, s->type == BCM_SYSPORT_STAT_NETDEV64) { do { start = u64_stats_fetch_begin(syncp); - data[i] = *(u64 *)p; + data[j] = *(u64 *)p; } while (u64_stats_fetch_retry(syncp, start)); } else - data[i] = *(u32 *)p; + data[j] = *(u32 *)p; j++; } diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c index 62bc9cae613c..622e89587e5d 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c @@ -5733,7 +5733,8 @@ const struct ethtool_ops bnxt_ethtool_ops = { .op_needs_rtnl = ETHTOOL_OP_NEEDS_RTNL_SCHANNELS | ETHTOOL_OP_NEEDS_RTNL_SRINGPARAM | ETHTOOL_OP_NEEDS_RTNL_SCOALESCE | - ETHTOOL_OP_NEEDS_RTNL_RSS, + ETHTOOL_OP_NEEDS_RTNL_RSS | + ETHTOOL_OP_NEEDS_RTNL_TEST, .supported_coalesce_params = ETHTOOL_COALESCE_USECS | ETHTOOL_COALESCE_MAX_FRAMES | ETHTOOL_COALESCE_USECS_IRQ | diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.c b/drivers/net/ethernet/broadcom/genet/bcmgenet.c index b916080f4ff1..e8908916558b 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.c +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.c @@ -57,6 +57,9 @@ */ #define GENET_RSB_PAD (sizeof(struct status_64) + 2) +/* RX buffer plus the skb_shared_info napi_build_skb() places behind it */ +#define GENET_RX_BUF_SIZE SKB_HEAD_ALIGN(RX_BUF_LENGTH) + /* Tx/Rx DMA register offset, skip 256 descriptors */ #define WORDS_PER_BD(p) (p->hw_params->words_per_bd) #define DMA_DESC_SIZE (WORDS_PER_BD(priv) * sizeof(u32)) @@ -852,7 +855,8 @@ static int bcmgenet_get_coalesce(struct net_device *dev, ec->rx_max_coalesced_frames = bcmgenet_rdma_ring_readl(priv, 0, DMA_MBUF_DONE_THRESH); ec->rx_coalesce_usecs = - bcmgenet_rdma_readl(priv, DMA_RING0_TIMEOUT) * 8192 / 1000; + (bcmgenet_rdma_readl(priv, DMA_RING0_TIMEOUT) & + DMA_TIMEOUT_MASK) * 8192 / 1000; for (i = 0; i <= priv->hw_params->rx_queues; i++) { ring = &priv->rx_rings[i]; @@ -1346,9 +1350,8 @@ static void bcmgenet_get_ethtool_stats(struct net_device *dev, p = (char *)&stats64; p += s->stat_offset; - if (sizeof(unsigned long) != sizeof(u32) && - s->stat_sizeof == sizeof(unsigned long)) - data[i] = *(unsigned long *)p; + if (s->stat_sizeof == sizeof(u64)) + data[i] = *(u64 *)p; else data[i] = *(u32 *)p; } @@ -1763,13 +1766,12 @@ static int bcmgenet_power_up(struct bcmgenet_priv *priv, int ret = 0; u32 reg; - if (!bcmgenet_has_ext(priv)) - return ret; - - reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT); - switch (mode) { case GENET_POWER_PASSIVE: + if (!bcmgenet_has_ext(priv)) + break; + + reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT); reg &= ~(EXT_PWR_DOWN_DLL | EXT_PWR_DOWN_BIAS | EXT_ENERGY_DET_MASK); if (GENET_IS_V5(priv) && !bcmgenet_has_ephy_16nm(priv)) { @@ -1793,8 +1795,12 @@ static int bcmgenet_power_up(struct bcmgenet_priv *priv, break; case GENET_POWER_CABLE_SENSE: + if (!bcmgenet_has_ext(priv)) + break; + /* enable APD */ if (!GENET_IS_V5(priv)) { + reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT); reg |= EXT_PWR_DN_EN_LD; bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT); } @@ -2251,11 +2257,12 @@ static int bcmgenet_rx_refill(struct bcmgenet_rx_ring *ring, struct enet_cb *cb) { struct bcmgenet_priv *priv = ring->priv; + unsigned int size = GENET_RX_BUF_SIZE; + unsigned int offset; dma_addr_t mapping; struct page *page; - page = page_pool_alloc_pages(ring->page_pool, - GFP_ATOMIC); + page = page_pool_dev_alloc(ring->page_pool, &offset, &size); if (!page) { priv->mib.alloc_rx_buff_failed++; netif_err(priv, rx_err, priv->dev, @@ -2264,9 +2271,13 @@ static int bcmgenet_rx_refill(struct bcmgenet_rx_ring *ring, } /* page_pool handles DMA mapping via PP_FLAG_DMA_MAP */ - mapping = page_pool_get_dma_addr(page); + mapping = page_pool_get_dma_addr(page) + offset; + dma_sync_single_for_device(&priv->pdev->dev, mapping, RX_BUF_LENGTH, + DMA_FROM_DEVICE); cb->rx_page = page; + cb->rx_offset = offset; + cb->rx_size = size; dmadesc_set_addr(priv, cb->bd_addr, mapping); return 0; @@ -2320,6 +2331,7 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring, while ((rxpktprocessed < rxpkttoprocess) && (rxpktprocessed < budget)) { + unsigned int rx_offset, rx_size; struct status_64 *status; struct page *rx_page; void *hard_start; @@ -2329,6 +2341,8 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring, /* Save the received page before refilling */ rx_page = cb->rx_page; + rx_offset = cb->rx_offset; + rx_size = cb->rx_size; if (bcmgenet_rx_refill(ring, cb)) { BCMGENET_STATS64_INC(stats, dropped); @@ -2338,10 +2352,10 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring, /* Sync the full buffer; the HW may have written anywhere * up to RX_BUF_LENGTH. */ - page_pool_dma_sync_for_cpu(ring->page_pool, rx_page, 0, + page_pool_dma_sync_for_cpu(ring->page_pool, rx_page, rx_offset, RX_BUF_LENGTH); - hard_start = page_address(rx_page); + hard_start = page_address(rx_page) + rx_offset; status = (struct status_64 *)hard_start; dma_length_status = status->length_status; @@ -2407,7 +2421,7 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring, /* Build SKB from the page - data starts at hard_start, * frame begins after RSB(64) + pad(2) = 66 bytes. */ - skb = napi_build_skb(hard_start, PAGE_SIZE); + skb = napi_build_skb(hard_start, rx_size); if (unlikely(!skb)) { BCMGENET_STATS64_INC(stats, dropped); page_pool_put_full_page(ring->page_pool, rx_page, @@ -2759,14 +2773,16 @@ static void bcmgenet_init_tx_ring(struct bcmgenet_priv *priv, static int bcmgenet_rx_ring_create_pool(struct bcmgenet_priv *priv, struct bcmgenet_rx_ring *ring) { + /* Buffers share a page. bcmgenet_rx_refill() syncs each one for the + * device, PP_FLAG_DMA_SYNC_DEV would sync the whole page. + */ struct page_pool_params pp_params = { .order = 0, - .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, + .flags = PP_FLAG_DMA_MAP, .pool_size = ring->size, .nid = NUMA_NO_NODE, .dev = &priv->pdev->dev, .dma_dir = DMA_FROM_DEVICE, - .max_len = RX_BUF_LENGTH, }; int err; @@ -3441,6 +3457,8 @@ static void bcmgenet_netif_stop(struct net_device *dev, bool stop_phy) { struct bcmgenet_priv *priv = netdev_priv(dev); + /* Stop completion polling before it can wake a stopped queue */ + bcmgenet_disable_tx_napi(priv); netif_tx_disable(dev); /* Disable MAC receive */ @@ -3455,7 +3473,6 @@ static void bcmgenet_netif_stop(struct net_device *dev, bool stop_phy) /* Disable MAC transmit. TX DMA disabled must be done before this */ umac_enable_set(priv, CMD_TX_EN, false); - bcmgenet_disable_tx_napi(priv); bcmgenet_disable_rx_napi(priv); bcmgenet_intr_disable(priv); @@ -3632,6 +3649,9 @@ static int bcmgenet_set_mac_addr(struct net_device *dev, void *p) if (netif_running(dev)) return -EBUSY; + if (!is_valid_ether_addr(addr->sa_data)) + return -EADDRNOTAVAIL; + eth_hw_addr_set(dev, addr->sa_data); return 0; @@ -4131,14 +4151,17 @@ static int bcmgenet_probe(struct platform_device *pdev) netif_set_real_num_rx_queues(priv->dev, priv->hw_params->rx_queues + 1); /* Set default coalescing parameters */ - for (i = 0; i <= priv->hw_params->rx_queues; i++) + for (i = 0; i <= priv->hw_params->rx_queues; i++) { + priv->rx_rings[i].priv = priv; + priv->rx_rings[i].index = i; priv->rx_rings[i].rx_max_coalesced_frames = 1; + } /* Initialize u64 stats seq counter for 32bit machines */ - for (i = 0; i <= priv->hw_params->rx_queues; i++) + for (i = 0; i <= GENET_MAX_MQ_CNT; i++) { u64_stats_init(&priv->rx_rings[i].stats64.syncp); - for (i = 0; i <= priv->hw_params->tx_queues; i++) u64_stats_init(&priv->tx_rings[i].stats64.syncp); + } /* libphy will determine the link state */ netif_carrier_off(dev); @@ -4320,6 +4343,8 @@ static int bcmgenet_suspend(struct device *d) netif_device_detach(dev); if (device_may_wakeup(d) && priv->wolopts) { + /* Stop completion polling before it can wake a stopped queue */ + bcmgenet_disable_tx_napi(priv); netif_tx_disable(dev); /* Suspend non-wake Rx data flows */ @@ -4348,7 +4373,6 @@ static int bcmgenet_suspend(struct device *d) netdev_warn(priv->dev, "Timed out while disabling TX DMA\n"); - bcmgenet_disable_tx_napi(priv); bcmgenet_disable_rx_napi(priv); disable_irq(priv->irq1); bcmgenet_tx_reclaim_all(dev); diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.h b/drivers/net/ethernet/broadcom/genet/bcmgenet.h index 22a958ba9902..86f2aed20dbe 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.h +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.h @@ -470,6 +470,8 @@ struct bcmgenet_rx_stats64 { struct enet_cb { struct sk_buff *skb; struct page *rx_page; + unsigned int rx_offset; + unsigned int rx_size; void __iomem *bd_addr; DEFINE_DMA_UNMAP_ADDR(dma_addr); DEFINE_DMA_UNMAP_LEN(dma_len); diff --git a/drivers/net/ethernet/broadcom/tg3.c b/drivers/net/ethernet/broadcom/tg3.c index 73a4b569b03e..8b6806a79edf 100644 --- a/drivers/net/ethernet/broadcom/tg3.c +++ b/drivers/net/ethernet/broadcom/tg3.c @@ -17915,11 +17915,14 @@ static int tg3_init_one(struct pci_dev *pdev, err = tg3_get_device_address(tp, addr); if (err) { - dev_err(&pdev->dev, - "Could not obtain valid ethernet address, aborting\n"); - goto err_out_apeunmap; + dev_warn_probe(&pdev->dev, err, + "Could not obtain a valid ethernet address\n"); + if (err == -EPROBE_DEFER) + goto err_out_apeunmap; + eth_hw_addr_random(dev); + } else { + eth_hw_addr_set(dev, addr); } - eth_hw_addr_set(dev, addr); intmbx = MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW; rcvmbx = MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW; @@ -18047,6 +18050,10 @@ static int tg3_init_one(struct pci_dev *pdev, return 0; err_out_apeunmap: + if (tg3_flag(tp, USE_PHYLIB)) + tg3_phy_fini(tp); + tg3_mdio_fini(tp); + if (tp->aperegs) { iounmap(tp->aperegs); tp->aperegs = NULL; diff --git a/drivers/net/ethernet/brocade/bna/bnad.c b/drivers/net/ethernet/brocade/bna/bnad.c index 8b75004ba7c9..55dfd4896785 100644 --- a/drivers/net/ethernet/brocade/bna/bnad.c +++ b/drivers/net/ethernet/brocade/bna/bnad.c @@ -2571,6 +2571,22 @@ bnad_ioceth_disable(struct bnad *bnad) return err; } +/* + * The IOC timers rearm one another, so deleting one cannot stop a + * sibling callback from arming it again. Shut them down so a later + * mod_timer() is ignored. + */ +static void +bnad_ioc_timers_shutdown(struct bnad *bnad) +{ + struct bfa_ioc *ioc = &bnad->bna.ioceth.ioc; + + timer_shutdown_sync(&ioc->ioc_timer); + timer_shutdown_sync(&ioc->sem_timer); + timer_shutdown_sync(&ioc->hb_timer); + timer_shutdown_sync(&ioc->iocpf_timer); +} + static int bnad_ioceth_enable(struct bnad *bnad) { @@ -3727,9 +3743,7 @@ bnad_pci_probe(struct pci_dev *pdev, bnad_res_free(bnad, &bnad->mod_res_info[0], BNA_MOD_RES_T_MAX); disable_ioceth: bnad_ioceth_disable(bnad); - timer_delete_sync(&bnad->bna.ioceth.ioc.ioc_timer); - timer_delete_sync(&bnad->bna.ioceth.ioc.sem_timer); - timer_delete_sync(&bnad->bna.ioceth.ioc.hb_timer); + bnad_ioc_timers_shutdown(bnad); spin_lock_irqsave(&bnad->bna_lock, flags); bna_uninit(bna); spin_unlock_irqrestore(&bnad->bna_lock, flags); @@ -3770,9 +3784,7 @@ bnad_pci_remove(struct pci_dev *pdev) mutex_lock(&bnad->conf_mutex); bnad_ioceth_disable(bnad); - timer_delete_sync(&bnad->bna.ioceth.ioc.ioc_timer); - timer_delete_sync(&bnad->bna.ioceth.ioc.sem_timer); - timer_delete_sync(&bnad->bna.ioceth.ioc.hb_timer); + bnad_ioc_timers_shutdown(bnad); spin_lock_irqsave(&bnad->bna_lock, flags); bna_uninit(bna); spin_unlock_irqrestore(&bnad->bna_lock, flags); diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index b8234ac4b602..8e5c034dc3a4 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -2749,14 +2749,24 @@ static int macb_alloc(struct macb *bp) size = bp->num_queues * macb_tx_ring_size_per_queue(bp); tx = dma_alloc_coherent(dev, size, &tx_dma, GFP_KERNEL); - if (!tx || upper_32_bits(tx_dma) != upper_32_bits(tx_dma + size - 1)) + if (!tx) + goto out_err; + /* Record the buffer so that the error path frees it. */ + bp->queues[0].tx_ring = tx; + bp->queues[0].tx_ring_dma = tx_dma; + if (upper_32_bits(tx_dma) != upper_32_bits(tx_dma + size - 1)) goto out_err; netdev_dbg(bp->netdev, "Allocated %zu bytes for %u TX rings at %08lx (mapped %p)\n", size, bp->num_queues, (unsigned long)tx_dma, tx); size = bp->num_queues * macb_rx_ring_size_per_queue(bp); rx = dma_alloc_coherent(dev, size, &rx_dma, GFP_KERNEL); - if (!rx || upper_32_bits(rx_dma) != upper_32_bits(rx_dma + size - 1)) + if (!rx) + goto out_err; + /* Record the buffer so that the error path frees it. */ + bp->queues[0].rx_ring = rx; + bp->queues[0].rx_ring_dma = rx_dma; + if (upper_32_bits(rx_dma) != upper_32_bits(rx_dma + size - 1)) goto out_err; netdev_dbg(bp->netdev, "Allocated %zu bytes for %u RX rings at %08lx (mapped %p)\n", size, bp->num_queues, (unsigned long)rx_dma, rx); diff --git a/drivers/net/ethernet/freescale/fman/fman.c b/drivers/net/ethernet/freescale/fman/fman.c index 299bab043175..46cc28895e56 100644 --- a/drivers/net/ethernet/freescale/fman/fman.c +++ b/drivers/net/ethernet/freescale/fman/fman.c @@ -2734,6 +2734,7 @@ static struct fman *read_dts_node(struct platform_device *of_dev) } clk_rate = clk_get_rate(clk); + clk_put(clk); if (!clk_rate) { err = -EINVAL; dev_err(&of_dev->dev, "%s: Failed to determine FM%d clock rate\n", diff --git a/drivers/net/ethernet/google/gve/gve_desc_dqo.h b/drivers/net/ethernet/google/gve/gve_desc_dqo.h index f7786b03c744..d2c86c8eeae2 100644 --- a/drivers/net/ethernet/google/gve/gve_desc_dqo.h +++ b/drivers/net/ethernet/google/gve/gve_desc_dqo.h @@ -14,6 +14,11 @@ #define GVE_TX_MAX_HDR_SIZE_DQO 255 #define GVE_TX_MIN_TSO_MSS_DQO 88 +/* HW limit. This also has to fit in the 14 bits of the mss field of + * struct gve_tx_tso_context_desc_dqo. + */ +#define GVE_TX_MAX_TSO_MSS_DQO 9728 + #ifndef __LITTLE_ENDIAN_BITFIELD #error "Only little endian supported" #endif diff --git a/drivers/net/ethernet/google/gve/gve_tx_dqo.c b/drivers/net/ethernet/google/gve/gve_tx_dqo.c index 80ab0a449ff5..ad99cbb77e29 100644 --- a/drivers/net/ethernet/google/gve/gve_tx_dqo.c +++ b/drivers/net/ethernet/google/gve/gve_tx_dqo.c @@ -577,40 +577,43 @@ static int gve_prep_tso(struct sk_buff *skb) int header_len; int err; - /* Note: HW requires MSS (gso_size) to be <= 9728 and the total length - * of the TSO to be <= 262143. + /* Note: HW requires the total length of the TSO to be <= 262143, + * this is enforced by netif_set_tso_max_size(). * - * However, we don't validate these because: - * - Hypervisor enforces a limit of 9K MTU - * - Kernel will not produce a TSO larger than 64k + * MSS (gso_size) can not be trusted: packets forwarded from a tap or + * injected by a packet socket can carry an arbitrary value, while the + * mss field of the TSO context descriptor is only 14 bits wide. + * + * A too big MSS is dropped here instead of being rejected from + * gve_features_check_dqo(), because software segmentation would + * produce packets larger than the device can send. */ - - if (unlikely(shinfo->gso_size < GVE_TX_MIN_TSO_MSS_DQO)) + if (unlikely(shinfo->gso_size > GVE_TX_MAX_TSO_MSS_DQO)) return -1; /* Needed because we will modify header. */ err = skb_cow_head(skb, 0); if (err < 0) return err; + shinfo = skb_shinfo(skb); l4_start = skb_transport_offset(skb); paylen = skb->len - l4_start; - switch (shinfo->gso_type) { - case SKB_GSO_TCPV4: - case SKB_GSO_TCPV6: + /* gso_type is a bitmask: SKB_GSO_DODGY, SKB_GSO_TCP_ECN and + * SKB_GSO_TCP_FIXEDID may be set alongside the protocol bit. + */ + if (shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) { tcp = tcp_hdr(skb); csum_replace_by_diff(&tcp->check, (__force __wsum)htonl(paylen)); header_len = skb_tcp_all_headers(skb); - break; - case SKB_GSO_UDP_L4: + } else if (shinfo->gso_type & SKB_GSO_UDP_L4) { udp = udp_hdr(skb); csum_replace_by_diff(&udp->check, (__force __wsum)htonl(paylen)); header_len = sizeof(struct udphdr) + l4_start; - break; - default: + } else { return -EINVAL; } @@ -918,13 +921,22 @@ static bool gve_can_send_tso(const struct sk_buff *skb) { const int max_bufs_per_seg = GVE_TX_MAX_DATA_DESCS - 1; const struct skb_shared_info *shinfo = skb_shinfo(skb); - const int header_len = skb_tcp_all_headers(skb); const int gso_size = shinfo->gso_size; int cur_seg_num_bufs; int prev_frag_size; int cur_seg_size; + int header_len; int i; + if (unlikely(gso_size < GVE_TX_MIN_TSO_MSS_DQO)) + return false; + + /* Must match the header length programmed by gve_prep_tso(). */ + if (skb_is_gso_tcp(skb)) + header_len = skb_tcp_all_headers(skb); + else + header_len = skb_transport_offset(skb) + sizeof(struct udphdr); + cur_seg_size = skb_headlen(skb) - header_len; prev_frag_size = skb_headlen(skb); cur_seg_num_bufs = cur_seg_size > 0; @@ -966,7 +978,17 @@ netdev_features_t gve_features_check_dqo(struct sk_buff *skb, struct net_device *dev, netdev_features_t features) { - if (skb_is_gso(skb) && !gve_can_send_tso(skb)) + if (!skb_is_gso(skb)) + return features; + + /* Keep the GSO bits for a too big MSS, so that gve_prep_tso() drops + * the packet: software segmentation would give packets larger than + * the device can send. + */ + if (skb_shinfo(skb)->gso_size > GVE_TX_MAX_TSO_MSS_DQO) + return features; + + if (!gve_can_send_tso(skb)) return features & ~NETIF_F_GSO_MASK; return features; diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c index bc6b269be299..f1cb6d56e4b9 100644 --- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c +++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c @@ -793,6 +793,7 @@ static int hns_mac_register_phy(struct hns_mac_cb *mac_cb) dev_err(mac_cb->dev, "mac%d mdio is NULL, dsaf will probe again later\n", mac_cb->mac_id); + put_device(&pdev->dev); return -EPROBE_DEFER; } @@ -801,6 +802,8 @@ static int hns_mac_register_phy(struct hns_mac_cb *mac_cb) dev_dbg(mac_cb->dev, "mac%d register phy addr:%d\n", mac_cb->mac_id, addr); + put_device(&pdev->dev); + return rc; } diff --git a/drivers/net/ethernet/ibm/emac/core.c b/drivers/net/ethernet/ibm/emac/core.c index 1d46cf6c2c12..e7043523457c 100644 --- a/drivers/net/ethernet/ibm/emac/core.c +++ b/drivers/net/ethernet/ibm/emac/core.c @@ -3044,6 +3044,15 @@ static int emac_probe(struct platform_device *ofdev) if (err) goto err_gone; + if (emac_phy_supports_gige(dev->phy_mode)) { + ndev->netdev_ops = &emac_gige_netdev_ops; + dev->commac.ops = &emac_commac_sg_ops; + } else { + ndev->netdev_ops = &emac_netdev_ops; + dev->commac.ops = &emac_commac_ops; + } + ndev->ethtool_ops = &emac_ethtool_ops; + dev->emacp = devm_platform_ioremap_resource(ofdev, 0); if (IS_ERR(dev->emacp)) { err = PTR_ERR(dev->emacp); @@ -3076,7 +3085,6 @@ static int emac_probe(struct platform_device *ofdev) dev->mdio_instance = platform_get_drvdata(dev->mdio_dev); /* Register with MAL */ - dev->commac.ops = &emac_commac_ops; dev->commac.dev = dev; dev->commac.tx_chan_mask = MAL_CHAN_MASK(dev->mal_tx_chan); dev->commac.rx_chan_mask = MAL_CHAN_MASK(dev->mal_rx_chan); @@ -3144,12 +3152,6 @@ static int emac_probe(struct platform_device *ofdev) ndev->features |= ndev->hw_features | NETIF_F_RXCSUM; } ndev->watchdog_timeo = 5 * HZ; - if (emac_phy_supports_gige(dev->phy_mode)) { - ndev->netdev_ops = &emac_gige_netdev_ops; - dev->commac.ops = &emac_commac_sg_ops; - } else - ndev->netdev_ops = &emac_netdev_ops; - ndev->ethtool_ops = &emac_ethtool_ops; /* MTU range: 46 - 1500 or whatever is in OF */ ndev->min_mtu = EMAC_MIN_MTU; diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c index 543e566425c1..4f479b6f731e 100644 --- a/drivers/net/ethernet/marvell/mvneta.c +++ b/drivers/net/ethernet/marvell/mvneta.c @@ -2340,7 +2340,7 @@ mvneta_swbm_rx_frame(struct mvneta_port *pp, /* Prefetch header */ prefetch(data); - xdp_buff_clear_frags_flag(xdp); + xdp_reinit_buff(xdp); xdp_prepare_buff(xdp, data, pp->rx_offset_correction + MVNETA_MH_SIZE, data_len, true); } diff --git a/drivers/net/ethernet/marvell/octeontx2/af/common.h b/drivers/net/ethernet/marvell/octeontx2/af/common.h index 779413a383b7..78e42549d990 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/common.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/common.h @@ -7,6 +7,10 @@ #ifndef COMMON_H #define COMMON_H +#include +#include +#include + #include "rvu_struct.h" #define OTX2_ALIGN 128 /* Align to cacheline */ @@ -44,6 +48,33 @@ struct qmem { u32 qsize; }; +static inline void *otx2_dma_alloc_coherent(struct device *dev, size_t size, + dma_addr_t *dma_handle) +{ + dma_addr_t dma_addr; + void *vaddr; + + vaddr = kzalloc(size, GFP_KERNEL); + if (!vaddr) + return NULL; + + dma_addr = dma_map_single(dev, vaddr, size, DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, dma_addr)) { + kfree(vaddr); + return NULL; + } + + *dma_handle = dma_addr; + return vaddr; +} + +static inline void otx2_dma_free_coherent(struct device *dev, size_t size, + void *vaddr, dma_addr_t dma_handle) +{ + dma_unmap_single(dev, dma_handle, size, DMA_BIDIRECTIONAL); + kfree(vaddr); +} + static inline int qmem_alloc(struct device *dev, struct qmem **q, int qsize, int entry_sz) { @@ -60,8 +91,11 @@ static inline int qmem_alloc(struct device *dev, struct qmem **q, qmem->entry_sz = entry_sz; qmem->alloc_sz = (qsize * entry_sz) + OTX2_ALIGN; - qmem->base = dma_alloc_attrs(dev, qmem->alloc_sz, &qmem->iova, - GFP_KERNEL, DMA_ATTR_FORCE_CONTIGUOUS); + + if (get_order(PAGE_ALIGN(qmem->alloc_sz)) > MAX_PAGE_ORDER) + return -ENOMEM; + + qmem->base = otx2_dma_alloc_coherent(dev, qmem->alloc_sz, &qmem->iova); if (!qmem->base) return -ENOMEM; @@ -80,10 +114,9 @@ static inline void qmem_free(struct device *dev, struct qmem *qmem) return; if (qmem->base) - dma_free_attrs(dev, qmem->alloc_sz, - qmem->base - qmem->align, - qmem->iova - qmem->align, - DMA_ATTR_FORCE_CONTIGUOUS); + otx2_dma_free_coherent(dev, qmem->alloc_sz, + qmem->base - qmem->align, + qmem->iova - qmem->align); devm_kfree(dev, qmem); } diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs.c b/drivers/net/ethernet/marvell/octeontx2/af/mcs.c index 211c10aa5880..30c672d526a0 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/mcs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs.c @@ -314,7 +314,7 @@ int mcs_clear_all_stats(struct mcs *mcs, u16 pcifunc, int dir) } /* Clear SC stats */ - for (id = 0; id < map->secy.max; id++) { + for (id = 0; id < map->sc.max; id++) { if (map->sc2pf_map[id] != pcifunc) continue; mcs_clear_stats(mcs, MCS_SC_STATS, id, dir); @@ -738,7 +738,7 @@ int mcs_free_all_rsrc(struct mcs *mcs, int dir, u16 pcifunc) } /* free sc entries */ - for (id = 0; id < map->secy.max; id++) { + for (id = 0; id < map->sc.max; id++) { if (map->sc2pf_map[id] != pcifunc) continue; mcs_free_rsrc(&map->sc, map->sc2pf_map, id, pcifunc); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c index 904374baae6f..2927633465d9 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c @@ -714,110 +714,72 @@ static int get_max_column_width(struct rvu *rvu) } /* Dumps current provisioning status of all RVU block LFs */ -static ssize_t rvu_dbg_rsrc_attach_status(struct file *filp, - char __user *buffer, - size_t count, loff_t *ppos) +static int rvu_dbg_rsrc_attach_status(struct seq_file *filp, void *unused) { - int index, off = 0, flag = 0, len = 0, i = 0; - struct rvu *rvu = filp->private_data; - int bytes_not_copied = 0; + struct rvu *rvu = filp->private; + int index, pf, vf, pcifunc; struct rvu_block block; - int pf, vf, pcifunc; - int buf_size = 2048; int lf_str_size; char *lfs; - char *buf; - /* don't allow partial reads */ - if (*ppos != 0) - return 0; - - buf = kzalloc(buf_size, GFP_KERNEL); - if (!buf) - return -ENOMEM; - - /* Get the maximum width of a column */ lf_str_size = get_max_column_width(rvu); + if (lf_str_size < 0) + return lf_str_size; lfs = kzalloc(lf_str_size, GFP_KERNEL); - if (!lfs) { - kfree(buf); + if (!lfs) return -ENOMEM; - } - off += scnprintf(&buf[off], buf_size - 1 - off, "%-*s", lf_str_size, - "pcifunc"); + + seq_printf(filp, "%-*s", lf_str_size, "pcifunc"); for (index = 0; index < BLK_COUNT; index++) - if (strlen(rvu->hw->block[index].name)) { - off += scnprintf(&buf[off], buf_size - 1 - off, - "%-*s", lf_str_size, - rvu->hw->block[index].name); - } + if (strlen(rvu->hw->block[index].name)) + seq_printf(filp, "%-*s", lf_str_size, + rvu->hw->block[index].name); - off += scnprintf(&buf[off], buf_size - 1 - off, "\n"); - bytes_not_copied = copy_to_user(buffer + (i * off), buf, off); - if (bytes_not_copied) - goto out; - - i++; - *ppos += off; + seq_putc(filp, '\n'); for (pf = 0; pf < rvu->hw->total_pfs; pf++) { for (vf = 0; vf <= rvu->hw->total_vfs; vf++) { - off = 0; - flag = 0; pcifunc = rvu_make_pcifunc(rvu->pdev, pf, vf); if (!pcifunc) continue; - if (vf) { - sprintf(lfs, "PF%d:VF%d", pf, vf - 1); - off = scnprintf(&buf[off], - buf_size - 1 - off, - "%-*s", lf_str_size, lfs); - } else { - sprintf(lfs, "PF%d", pf); - off = scnprintf(&buf[off], - buf_size - 1 - off, - "%-*s", lf_str_size, lfs); - } - for (index = 0; index < BLK_COUNT; index++) { block = rvu->hw->block[index]; if (!strlen(block.name)) continue; - len = 0; - lfs[len] = '\0'; + lfs[0] = '\0'; get_lf_str_list(&block, pcifunc, lfs); if (strlen(lfs)) - flag = 1; - - off += scnprintf(&buf[off], buf_size - 1 - off, - "%-*s", lf_str_size, lfs); + break; } - if (flag) { - off += scnprintf(&buf[off], - buf_size - 1 - off, "\n"); - bytes_not_copied = copy_to_user(buffer + - (i * off), - buf, off); - if (bytes_not_copied) - goto out; + if (index == BLK_COUNT) + continue; - i++; - *ppos += off; + if (vf) + sprintf(lfs, "PF%d:VF%d", pf, vf - 1); + else + sprintf(lfs, "PF%d", pf); + seq_printf(filp, "%-*s", lf_str_size, lfs); + + for (index = 0; index < BLK_COUNT; index++) { + block = rvu->hw->block[index]; + if (!strlen(block.name)) + continue; + + lfs[0] = '\0'; + get_lf_str_list(&block, pcifunc, lfs); + seq_printf(filp, "%-*s", lf_str_size, lfs); } + seq_putc(filp, '\n'); } } -out: kfree(lfs); - kfree(buf); - if (bytes_not_copied) - return -EFAULT; - return *ppos; + return 0; } -RVU_DEBUG_FOPS(rsrc_status, rsrc_attach_status, NULL); +RVU_DEBUG_SEQ_FOPS(rsrc_status, rsrc_attach_status, NULL); static int rvu_dbg_rvu_pf_cgx_map_display(struct seq_file *filp, void *unused) { diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c index 8e41431c7f9c..e92b77789f08 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c @@ -242,15 +242,6 @@ int cn20k_register_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs) #define RQ_BP_LVL_AURA (255 - ((85 * 256) / 100)) /* BP when 85% is full */ -static u8 cn20k_aura_bpid_idx(struct otx2_nic *pfvf, int aura_id) -{ -#ifdef CONFIG_DCB - return pfvf->queue_to_pfc_map[aura_id]; -#else - return 0; -#endif -} - static int cn20k_tc_get_entry_index(struct otx2_flow_config *flow_cfg, struct otx2_tc_flow *node) { @@ -580,7 +571,7 @@ static int cn20k_aura_aq_init(struct otx2_nic *pfvf, int aura_id, if (pfvf->nix_blkaddr == BLKADDR_NIX1) aq->aura.bp_ena = 1; - bpid_idx = cn20k_aura_bpid_idx(pfvf, aura_id); + bpid_idx = otx2_get_bpid_idx(pfvf, aura_id); aq->aura.bpid = pfvf->bpid[bpid_idx]; /* Set backpressure level for RQ's Aura */ diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c index 175992188c18..b11389a841d1 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c @@ -394,7 +394,7 @@ int otx2_rss_init(struct otx2_nic *pfvf) struct otx2_rss_info *rss = &pfvf->hw.rss_info; int idx, ret = 0; - rss->rss_size = sizeof(*rss->ind_tbl); + rss->rss_size = ARRAY_SIZE(rss->ind_tbl); /* Init RSS key if it is not setup already */ if (!rss->enable) @@ -1055,6 +1055,7 @@ int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx) struct nix_aq_enq_req *aq; struct otx2_cq_queue *cq; struct otx2_pool *pool; + u8 bpid_idx; cq = &qset->cq[qidx]; cq->cq_idx = qidx; @@ -1132,11 +1133,8 @@ int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx) if (!is_otx2_lbkvf(pfvf->pdev)) { /* Enable receive CQ backpressure */ aq->cq.bp_ena = 1; -#ifdef CONFIG_DCB - aq->cq.bpid = pfvf->bpid[pfvf->queue_to_pfc_map[qidx]]; -#else - aq->cq.bpid = pfvf->bpid[0]; -#endif + bpid_idx = otx2_get_bpid_idx(pfvf, qidx); + aq->cq.bpid = pfvf->bpid[bpid_idx]; /* Set backpressure level is same as cq pass level */ aq->cq.bp = RQ_PASS_LVL_CQ(pfvf->hw.rq_skid, qset->rqe_cnt); @@ -1378,6 +1376,7 @@ int otx2_aura_aq_init(struct otx2_nic *pfvf, int aura_id, { struct npa_aq_enq_req *aq; struct otx2_pool *pool; + u8 bpid_idx; int err; pool = &pfvf->qset.pool[pool_id]; @@ -1433,11 +1432,8 @@ int otx2_aura_aq_init(struct otx2_nic *pfvf, int aura_id, */ if (pfvf->nix_blkaddr == BLKADDR_NIX1) aq->aura.bp_ena = 1; -#ifdef CONFIG_DCB - aq->aura.nix0_bpid = pfvf->bpid[pfvf->queue_to_pfc_map[aura_id]]; -#else - aq->aura.nix0_bpid = pfvf->bpid[0]; -#endif + bpid_idx = otx2_get_bpid_idx(pfvf, aura_id); + aq->aura.nix0_bpid = pfvf->bpid[bpid_idx]; /* Set backpressure level for RQ's Aura */ aq->aura.bp = RQ_BP_LVL_AURA; diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h index eecee612b7b2..44cd56a0c2c4 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h @@ -1029,6 +1029,15 @@ static inline int otx2_tc_flower_rule_cnt(struct otx2_nic *pfvf) return pfvf->flow_cfg->nr_flows; } +static inline u8 otx2_get_bpid_idx(struct otx2_nic *pfvf, int qidx) +{ +#ifdef CONFIG_DCB + if (pfvf->queue_to_pfc_map) + return pfvf->queue_to_pfc_map[qidx]; +#endif + return 0; +} + /* MSI-X APIs */ void otx2_free_cints(struct otx2_nic *pfvf, int n); void otx2_set_cints_affinity(struct otx2_nic *pfvf); diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c index fd7a49ae88d0..ad0fdc4133fb 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c @@ -4509,6 +4509,10 @@ static int mtk_unreg_dev(struct mtk_eth *eth) mac = netdev_priv(eth->netdev[i]); if (MTK_HAS_CAPS(eth->soc->caps, MTK_QDMA)) unregister_netdevice_notifier(&mac->device_notifier); + + if (eth->netdev[i]->reg_state != NETREG_REGISTERED) + continue; + unregister_netdev(eth->netdev[i]); } @@ -5337,22 +5341,12 @@ static int mtk_probe(struct platform_device *pdev) } } - for (i = 0; i < MTK_MAX_DEVS; i++) { - if (!eth->netdev[i]) - continue; - - err = register_netdev(eth->netdev[i]); - if (err) { - dev_err(eth->dev, "error bringing up device\n"); - goto err_deinit_ppe; - } else - netif_info(eth, probe, eth->netdev[i], - "mediatek frame engine at 0x%08lx, irq %d\n", - eth->netdev[i]->base_addr, eth->irq[MTK_FE_IRQ_SHARED]); - } - /* we run 2 devices on the same DMA ring so we need a dummy device - * for NAPI to work + * for NAPI to work. Allocate it before registering the netdevs so a + * concurrent ndo_open (e.g. netifd bringing the first netdev up the + * instant it is registered) never observes eth->dummy_dev == NULL in + * mtk_dma_init() -> mtk_rx_alloc() -> __xdp_rxq_info_reg() (net/core/xdp.c + * "Missing net_device from driver") and fails the first open. */ eth->dummy_dev = alloc_netdev_dummy(0); if (!eth->dummy_dev) { @@ -5363,6 +5357,20 @@ static int mtk_probe(struct platform_device *pdev) netif_napi_add(eth->dummy_dev, ð->tx_napi, mtk_napi_tx); netif_napi_add(eth->dummy_dev, ð->rx_napi, mtk_napi_rx); + for (i = 0; i < MTK_MAX_DEVS; i++) { + if (!eth->netdev[i]) + continue; + + err = register_netdev(eth->netdev[i]); + if (err) { + dev_err(eth->dev, "error bringing up device\n"); + goto err_unreg_netdev; + } else + netif_info(eth, probe, eth->netdev[i], + "mediatek frame engine at 0x%08lx, irq %d\n", + eth->netdev[i]->base_addr, eth->irq[MTK_FE_IRQ_SHARED]); + } + platform_set_drvdata(pdev, eth); schedule_delayed_work(ð->reset.monitor_work, MTK_DMA_MONITOR_TIMEOUT); @@ -5372,6 +5380,16 @@ static int mtk_probe(struct platform_device *pdev) err_unreg_netdev: mtk_unreg_dev(eth); err_deinit_ppe: + /* A netdev opened while probe was still registering can have queued + * the frame-engine reset worker, which takes rtnl and drives the + * hardware; cancel it before anything below is torn down. + */ + cancel_work_sync(ð->pending_work); + if (eth->dummy_dev) { + netif_napi_del(ð->tx_napi); + netif_napi_del(ð->rx_napi); + free_netdev(eth->dummy_dev); + } mtk_ppe_deinit(eth); mtk_mdio_cleanup(eth); err_free_dev: diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c index baac38bece14..4b7b0a0fc2b2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c @@ -85,9 +85,16 @@ mlx5_esw_bridge_lower_rep_vport_num_vhca_id_get(struct net_device *dev, struct m struct net_device *lower_dev; struct list_head *iter; - if (netif_is_lag_master(dev) || mlx5e_eswitch_rep(dev)) - return mlx5_esw_bridge_rep_vport_num_vhca_id_get(dev, esw, vport_num, - esw_owner_vhca_id); + if (netif_is_lag_master(dev) || mlx5e_eswitch_rep(dev)) { + struct net_device *rep; + + rep = mlx5_esw_bridge_rep_vport_num_vhca_id_get(dev, esw, vport_num, + esw_owner_vhca_id); + if (rep && !mlx5_esw_bridge_port_exists(*vport_num, *esw_owner_vhca_id, + esw->br_offloads)) + return NULL; + return rep; + } netdev_for_each_lower_dev(dev, lower_dev, iter) { struct net_device *rep; @@ -104,6 +111,28 @@ mlx5_esw_bridge_lower_rep_vport_num_vhca_id_get(struct net_device *dev, struct m return NULL; } +static bool mlx5_esw_bridge_rep_port_lookup(struct net_device *dev, + struct mlx5_esw_bridge_offloads *br_offloads, + u16 *vport_num, u16 *esw_owner_vhca_id) +{ + if (!mlx5_esw_bridge_rep_vport_num_vhca_id_get(dev, br_offloads->esw, vport_num, + esw_owner_vhca_id)) + return false; + + return mlx5_esw_bridge_port_exists(*vport_num, *esw_owner_vhca_id, br_offloads); +} + +static bool mlx5_esw_bridge_lower_rep_port_lookup(struct net_device *dev, + struct mlx5_esw_bridge_offloads *br_offloads, + u16 *vport_num, u16 *esw_owner_vhca_id) +{ + if (!mlx5_esw_bridge_lower_rep_vport_num_vhca_id_get(dev, br_offloads->esw, vport_num, + esw_owner_vhca_id)) + return false; + + return mlx5_esw_bridge_port_exists(*vport_num, *esw_owner_vhca_id, br_offloads); +} + static bool mlx5_esw_bridge_is_local(struct net_device *dev, struct net_device *rep, struct mlx5_eswitch *esw) { @@ -218,8 +247,7 @@ mlx5_esw_bridge_port_obj_add(struct net_device *dev, u16 vport_num, esw_owner_vhca_id; int err; - if (!mlx5_esw_bridge_rep_vport_num_vhca_id_get(dev, br_offloads->esw, &vport_num, - &esw_owner_vhca_id)) + if (!mlx5_esw_bridge_rep_port_lookup(dev, br_offloads, &vport_num, &esw_owner_vhca_id)) return 0; port_obj_info->handled = true; @@ -251,8 +279,7 @@ mlx5_esw_bridge_port_obj_del(struct net_device *dev, const struct switchdev_obj_port_mdb *mdb; u16 vport_num, esw_owner_vhca_id; - if (!mlx5_esw_bridge_rep_vport_num_vhca_id_get(dev, br_offloads->esw, &vport_num, - &esw_owner_vhca_id)) + if (!mlx5_esw_bridge_rep_port_lookup(dev, br_offloads, &vport_num, &esw_owner_vhca_id)) return 0; port_obj_info->handled = true; @@ -283,8 +310,8 @@ mlx5_esw_bridge_port_obj_attr_set(struct net_device *dev, u16 vport_num, esw_owner_vhca_id; int err = 0; - if (!mlx5_esw_bridge_lower_rep_vport_num_vhca_id_get(dev, br_offloads->esw, &vport_num, - &esw_owner_vhca_id)) + if (!mlx5_esw_bridge_lower_rep_port_lookup(dev, br_offloads, &vport_num, + &esw_owner_vhca_id)) return 0; port_attr_info->handled = true; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c index 6c87a1c7db09..c1841ce74d9a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c @@ -1082,6 +1082,9 @@ mlx5_tc_ct_shared_counter_get(struct mlx5_tc_ct_priv *ct_priv, spin_unlock_bh(&ct_priv->ht_lock); + if (rev_entry) + mlx5_tc_ct_entry_put(rev_entry); + create_counter: shared_counter = mlx5_tc_ct_counter_create(ct_priv); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c index 77ea51bfbaae..2cca20e1b9b7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c @@ -293,6 +293,9 @@ static u64 mlx5e_xsk_fill_timestamp(void *_priv) struct mlx5e_xsk_tx_complete *priv = _priv; u64 ts; + if (!priv->cqe) + return 0; + ts = get_cqe_ts(priv->cqe); if (mlx5_is_real_time_rq(priv->cq->mdev) || mlx5_is_real_time_sq(priv->cq->mdev)) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index 329608c59313..8ffa8068e90a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -1564,14 +1564,14 @@ static void setup_fte_addr6(struct mlx5_flow_spec *spec, memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), saddr, 16); memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, - outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), dmask, 16); + outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), smask, 16); } if (!addr6_all_zero(daddr)) { memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), daddr, 16); memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, - outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), smask, 16); + outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), dmask, 16); } } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c index daff53ba7d09..38a3415acf7a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c @@ -1724,6 +1724,8 @@ void mlx5e_macsec_build_netdev(struct mlx5e_priv *priv) mlx5_core_dbg(priv->mdev, "mlx5e: MACsec acceleration enabled\n"); netdev->macsec_ops = &macsec_offload_ops; netdev->features |= NETIF_F_HW_MACSEC; + netdev->hw_features |= NETIF_F_HW_MACSEC; + netdev->vlan_features |= NETIF_F_HW_MACSEC; netif_keep_dst(netdev); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index fc110a7d16e8..4c6060d54fbe 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -5851,7 +5851,6 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev) netdev->vlan_features |= NETIF_F_SG; netdev->vlan_features |= NETIF_F_HW_CSUM; - netdev->vlan_features |= NETIF_F_HW_MACSEC; netdev->vlan_features |= NETIF_F_GRO; netdev->vlan_features |= NETIF_F_TSO; netdev->vlan_features |= NETIF_F_TSO6; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c index 87b5fd349594..b4cf3c5ac0dd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c @@ -1649,10 +1649,8 @@ int mlx5_esw_bridge_vport_unlink(struct net_device *br_netdev, u16 vport_num, int err; port = mlx5_esw_bridge_port_lookup(vport_num, esw_owner_vhca_id, br_offloads); - if (!port) { - NL_SET_ERR_MSG_MOD(extack, "Port is not attached to any bridge"); - return -EINVAL; - } + if (!port) + return 0; if (port->bridge->ifindex != br_netdev->ifindex) { NL_SET_ERR_MSG_MOD(extack, "Port is attached to another bridge"); return -EINVAL; @@ -1682,10 +1680,19 @@ int mlx5_esw_bridge_vport_peer_unlink(struct net_device *br_netdev, u16 vport_nu struct mlx5_esw_bridge_offloads *br_offloads, struct netlink_ext_ack *extack) { + if (!MLX5_CAP_ESW(br_offloads->esw->dev, merged_eswitch)) + return 0; + return mlx5_esw_bridge_vport_unlink(br_netdev, vport_num, esw_owner_vhca_id, br_offloads, extack); } +bool mlx5_esw_bridge_port_exists(u16 vport_num, u16 esw_owner_vhca_id, + struct mlx5_esw_bridge_offloads *br_offloads) +{ + return mlx5_esw_bridge_port_lookup(vport_num, esw_owner_vhca_id, br_offloads); +} + int mlx5_esw_bridge_port_vlan_add(u16 vport_num, u16 esw_owner_vhca_id, u16 vid, u16 flags, struct mlx5_esw_bridge_offloads *br_offloads, struct netlink_ext_ack *extack) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.h b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.h index d6f539161993..a4e59cc21089 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.h @@ -80,6 +80,8 @@ int mlx5_esw_bridge_vlan_proto_set(u16 vport_num, u16 esw_owner_vhca_id, u16 pro struct mlx5_esw_bridge_offloads *br_offloads); int mlx5_esw_bridge_mcast_set(u16 vport_num, u16 esw_owner_vhca_id, bool enable, struct mlx5_esw_bridge_offloads *br_offloads); +bool mlx5_esw_bridge_port_exists(u16 vport_num, u16 esw_owner_vhca_id, + struct mlx5_esw_bridge_offloads *br_offloads); int mlx5_esw_bridge_port_vlan_add(u16 vport_num, u16 esw_owner_vhca_id, u16 vid, u16 flags, struct mlx5_esw_bridge_offloads *br_offloads, struct netlink_ext_ack *extack); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c index c655f6e32e9b..3b34bec559e0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c @@ -1266,25 +1266,45 @@ void mlx5_lag_remove_devices(struct mlx5_lag *ldev) mlx5_lag_remove_devices_filter(ldev, MLX5_LAG_FILTER_PORTS); } +static int mlx5_lag_reload_ib_reps_idx(struct mlx5_lag *ldev, int idx, + u32 flags) +{ + struct lag_func *pf = mlx5_lag_pf(ldev, idx); + struct mlx5_eswitch *esw; + int ret; + + if (pf->dev->priv.flags & flags) + return 0; + + esw = pf->dev->priv.eswitch; + mlx5_esw_reps_block(esw); + ret = mlx5_eswitch_reload_ib_reps(esw); + mlx5_esw_reps_unblock(esw); + + return ret; +} + static int mlx5_lag_reload_ib_reps_unlocked(struct mlx5_lag *ldev, u32 flags, u32 filter, bool cont_on_fail) { - struct lag_func *pf; + int master_idx = mlx5_lag_get_dev_index_by_seq_filter(ldev, MLX5_LAG_P1, + filter); int ret; int i; - mlx5_lag_for_each(i, 0, ldev, filter) { - pf = mlx5_lag_pf(ldev, i); - if (!(pf->dev->priv.flags & flags)) { - struct mlx5_eswitch *esw; + if (master_idx < 0) + return -EINVAL; - esw = pf->dev->priv.eswitch; - mlx5_esw_reps_block(esw); - ret = mlx5_eswitch_reload_ib_reps(esw); - mlx5_esw_reps_unblock(esw); - if (ret && !cont_on_fail) - return ret; - } + ret = mlx5_lag_reload_ib_reps_idx(ldev, master_idx, flags); + if (ret && !cont_on_fail) + return ret; + + mlx5_lag_for_each(i, 0, ldev, filter) { + if (i == master_idx) + continue; + ret = mlx5_lag_reload_ib_reps_idx(ldev, i, flags); + if (ret && !cont_on_fail) + return ret; } return 0; @@ -1923,9 +1943,11 @@ static int mlx5_handle_changeupper_event(struct mlx5_lag *ldev, } } if (i < MLX5_MAX_PORTS) { - slave = bond_slave_get_rcu(ndev_tmp); - if (slave) - has_inactive |= bond_is_slave_inactive(slave); + if (netif_is_bond_master(upper)) { + slave = bond_slave_get_rcu(ndev_tmp); + if (slave) + has_inactive |= bond_is_slave_inactive(slave); + } bond_status |= (1 << idx); } @@ -2009,7 +2031,7 @@ static int mlx5_handle_changeinfodata_event(struct mlx5_lag *ldev, bool has_inactive = 0; int idx; - if (!netif_is_lag_master(ndev)) + if (!netif_is_bond_master(ndev)) return 0; rcu_read_lock(); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/shared_fdb.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/shared_fdb.c index 6b4ad3c53f2f..424040918fa3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/shared_fdb.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/shared_fdb.c @@ -270,6 +270,7 @@ int mlx5_lag_shared_fdb_create(struct mlx5_lag *ldev, pf->sd_fdb_active = false; } mlx5_lag_destroy_single_fdb_filter(ldev, group_id); + mlx5_lag_unload_reps_from_locked(ldev, filter); } err_add_devices: mlx5_lag_add_devices_filter(ldev, filter); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c index 64f92427602d..75855481522b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c @@ -37,6 +37,7 @@ struct mlx5_devcom_comp { struct mlx5_devcom_key key; mlx5_devcom_event_handler_t handler; struct kref ref; + int nr_devs; bool ready; struct rw_semaphore sem; struct lock_class_key lock_key; @@ -170,6 +171,7 @@ devcom_alloc_comp_dev(struct mlx5_devcom_dev *devc, down_write(&comp->sem); list_add_tail(&devcom->list, &comp->comp_dev_list_head); + WRITE_ONCE(comp->nr_devs, comp->nr_devs + 1); up_write(&comp->sem); return devcom; @@ -182,6 +184,7 @@ devcom_free_comp_dev(struct mlx5_devcom_comp_dev *devcom) down_write(&comp->sem); list_del(&devcom->list); + WRITE_ONCE(comp->nr_devs, comp->nr_devs - 1); up_write(&comp->sem); kref_put(&devcom->devc->ref, mlx5_devcom_dev_release); @@ -284,7 +287,7 @@ int mlx5_devcom_comp_get_size(struct mlx5_devcom_comp_dev *devcom) { struct mlx5_devcom_comp *comp = devcom->comp; - return kref_read(&comp->ref); + return READ_ONCE(comp->nr_devs); } int mlx5_devcom_locked_send_event(struct mlx5_devcom_comp_dev *devcom, diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h index 64b958df7774..baba3471bf5a 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h @@ -974,6 +974,7 @@ enum { /* PUL User Registers */ #define FBNIC_CSR_START_PUL_USER 0x31000 /* CSR section delimiter */ #define FBNIC_PUL_OB_TLP_HDR_AW_CFG 0x3103d /* 0xc40f4 */ +#define FBNIC_PUL_OB_TLP_HDR_AW_CFG_FLUSH_MODE CSR_BIT(20) #define FBNIC_PUL_OB_TLP_HDR_AW_CFG_FLUSH CSR_BIT(19) #define FBNIC_PUL_OB_TLP_HDR_AW_CFG_BME CSR_BIT(18) #define FBNIC_PUL_OB_TLP_HDR_AW_CFG_RDE_ATTR CSR_GENMASK(17, 15) @@ -1215,6 +1216,10 @@ enum { #define FBNIC_IPC_MBX_DESC_LEN_MASK DESC_GENMASK(63, 48) #define FBNIC_IPC_MBX_DESC_EOM DESC_BIT(46) #define FBNIC_IPC_MBX_DESC_ADDR_MASK DESC_GENMASK(45, 3) +/* Set with FW_CMPL when the FW completed a descriptor without successfully + * processing it (e.g. a mailbox DMA error); the completion has no valid data. + */ +#define FBNIC_IPC_MBX_DESC_FW_ERR DESC_BIT(2) #define FBNIC_IPC_MBX_DESC_FW_CMPL DESC_BIT(1) #define FBNIC_IPC_MBX_DESC_HOST_CMPL DESC_BIT(0) diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c b/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c index 3c4563c8f403..6edfa0aa69f1 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c @@ -539,8 +539,8 @@ static void fbnic_dbg_fw_mbx_display(struct seq_file *s, /* Generate header */ seq_puts(s, mbx_idx == FBNIC_IPC_MBX_RX_IDX ? "Rx\n" : "Tx\n"); - seq_printf(s, "Rdy: %d Head: %d Tail: %d\n", - mbx->ready, mbx->head, mbx->tail); + seq_printf(s, "Rdy: %d Head: %d Tail: %d resp_error: %llu\n", + mbx->ready, mbx->head, mbx->tail, mbx->resp_error); snprintf(hdr, sizeof(hdr), "%3s %-4s %s %-12s %s %-3s %-16s\n", "Idx", "Len", "E", "Addr", "F", "H", "Raw"); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c index 0e47088ec44b..76e9a545bb16 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c @@ -313,6 +313,11 @@ fbnic_get_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, kernel_ring->hds_thresh = fbn->hds_thresh; } +static u32 fbnic_ring_size_pow2(u32 size) +{ + return size ? roundup_pow_of_two(size) : 0; +} + static void fbnic_set_rings(struct fbnic_net *fbn, struct ethtool_ringparam *ring, struct kernel_ethtool_ringparam *kernel_ring) @@ -334,10 +339,10 @@ fbnic_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, struct fbnic_net *clone; int err; - ring->rx_pending = roundup_pow_of_two(ring->rx_pending); - ring->rx_mini_pending = roundup_pow_of_two(ring->rx_mini_pending); - ring->rx_jumbo_pending = roundup_pow_of_two(ring->rx_jumbo_pending); - ring->tx_pending = roundup_pow_of_two(ring->tx_pending); + ring->rx_pending = fbnic_ring_size_pow2(ring->rx_pending); + ring->rx_mini_pending = fbnic_ring_size_pow2(ring->rx_mini_pending); + ring->rx_jumbo_pending = fbnic_ring_size_pow2(ring->rx_jumbo_pending); + ring->tx_pending = fbnic_ring_size_pow2(ring->tx_pending); /* These are absolute minimums allowing the device and driver to operate * but not necessarily guarantee reasonable performance. Settings below @@ -2025,7 +2030,8 @@ static const struct ethtool_ops fbnic_ethtool_ops = { ETHTOOL_OP_NEEDS_RTNL_SPAUSEPARAM | ETHTOOL_OP_NEEDS_RTNL_SCHANNELS | ETHTOOL_OP_NEEDS_RTNL_SRINGPARAM | - ETHTOOL_OP_NEEDS_RTNL_GLINK, + ETHTOOL_OP_NEEDS_RTNL_GLINK | + ETHTOOL_OP_NEEDS_RTNL_TEST, .get_drvinfo = fbnic_get_drvinfo, .get_regs_len = fbnic_get_regs_len, .get_regs = fbnic_get_regs, diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw.c b/drivers/net/ethernet/meta/fbnic/fbnic_fw.c index 283d25fae79e..6d7eb8479edf 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_fw.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw.c @@ -60,8 +60,15 @@ static void fbnic_mbx_reset_desc_ring(struct fbnic_dev *fbd, int mbx_idx) */ switch (mbx_idx) { case FBNIC_IPC_MBX_RX_IDX: + /* Clearing BME blocks the device from writing to the host + * but leaves the requests parked in the write pipeline. The + * write path only clears outstanding requests when both FLUSH + * and FLUSH_MODE are set; FLUSH_MODE lets them drain without + * landing on the host. + */ wr32(fbd, FBNIC_PUL_OB_TLP_HDR_AW_CFG, - FBNIC_PUL_OB_TLP_HDR_AW_CFG_FLUSH); + FBNIC_PUL_OB_TLP_HDR_AW_CFG_FLUSH | + FBNIC_PUL_OB_TLP_HDR_AW_CFG_FLUSH_MODE); break; case FBNIC_IPC_MBX_TX_IDX: wr32(fbd, FBNIC_PUL_OB_TLP_HDR_AR_CFG, @@ -285,6 +292,12 @@ static void fbnic_mbx_process_tx_msgs(struct fbnic_dev *fbd) if (!(desc & FBNIC_IPC_MBX_DESC_FW_CMPL)) break; + if (desc & FBNIC_IPC_MBX_DESC_FW_ERR) { + tx_mbx->resp_error++; + dev_warn_ratelimited(fbd->dev, + "FW completed a Tx mailbox request with an error\n"); + } + fbnic_mbx_unmap_and_free_msg(fbd, FBNIC_IPC_MBX_TX_IDX, head); head++; @@ -1666,6 +1679,13 @@ static void fbnic_mbx_process_rx_msgs(struct fbnic_dev *fbd) if (!(desc & FBNIC_IPC_MBX_DESC_FW_CMPL)) break; + if (desc & FBNIC_IPC_MBX_DESC_FW_ERR) { + rx_mbx->resp_error++; + dev_warn_ratelimited(fbd->dev, + "FW reported an error on an Rx mailbox message; dropping\n"); + goto next_page; + } + dma_sync_single_for_cpu(fbd->dev, rx_mbx->buf_info[head].addr, FBNIC_RX_PAGE_SIZE, DMA_FROM_DEVICE); @@ -1733,7 +1753,9 @@ void fbnic_mbx_poll(struct fbnic_dev *fbd) int fbnic_mbx_poll_tx_ready(struct fbnic_dev *fbd) { struct fbnic_fw_mbx *tx_mbx = &fbd->mbx[FBNIC_IPC_MBX_TX_IDX]; + struct fbnic_fw_mbx *rx_mbx = &fbd->mbx[FBNIC_IPC_MBX_RX_IDX]; unsigned long timeout = jiffies + 10 * HZ + 1; + u64 tx_resp_error, rx_resp_error; int err, i; do { @@ -1764,6 +1786,9 @@ int fbnic_mbx_poll_tx_ready(struct fbnic_dev *fbd) * mgmt.version once we get the actual version from the firmware * in the capabilities request message. */ +send_cap_req: + tx_resp_error = tx_mbx->resp_error; + rx_resp_error = rx_mbx->resp_error; err = fbnic_fw_xmit_simple_msg(fbd, FBNIC_TLV_MSG_ID_HOST_CAP_REQ); if (err) goto clean_mbx; @@ -1781,9 +1806,27 @@ int fbnic_mbx_poll_tx_ready(struct fbnic_dev *fbd) msleep(20); fbnic_mbx_poll(fbd); + /* A valid capabilities response ends the poll. Check it + * before the FW_ERR retry below so a response parsed in the + * same poll as an unrelated FW_ERR is not discarded. + */ + if (fbd->fw_cap.running.mgmt.version >= MIN_FW_VER_CODE) + break; + /* set err, but wait till mgmt.version check to report it */ - if (!time_is_after_jiffies(timeout)) + if (!time_is_after_jiffies(timeout)) { err = -ETIMEDOUT; + continue; + } + + /* The FW can flag our capabilities request (Tx) or its + * response (Rx) with FW_ERR, in which case it produced no + * usable response. The ring is not wedged, so re-issue the + * request instead of spinning until the timeout. + */ + if (tx_mbx->resp_error != tx_resp_error || + rx_mbx->resp_error != rx_resp_error) + goto send_cap_req; } return 0; diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw.h b/drivers/net/ethernet/meta/fbnic/fbnic_fw.h index d84723e4cfa3..5f9969247e30 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_fw.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw.h @@ -13,6 +13,7 @@ struct fbnic_tlv_msg; struct fbnic_fw_mbx { u8 ready, head, tail; + u64 resp_error; struct { struct fbnic_tlv_msg *msg; dma_addr_t addr; diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c index 8b9bc9e8ea56..c6698e3002a1 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c @@ -434,6 +434,7 @@ static int fbnic_pm_suspend(struct device *dev) { struct fbnic_dev *fbd = dev_get_drvdata(dev); struct net_device *netdev = fbd->netdev; + struct fbnic_net *fbn; if (fbnic_init_failure(fbd)) goto null_uc_addr; @@ -441,11 +442,16 @@ static int fbnic_pm_suspend(struct device *dev) rtnl_lock(); netdev_lock(netdev); + fbn = netdev_priv(netdev); + netif_device_detach(netdev); if (netif_running(netdev)) netdev->netdev_ops->ndo_stop(netdev); + /* The IRQs are about to be freed, so drop the napi vector count */ + fbn->num_napi = 0; + netdev_unlock(netdev); rtnl_unlock(); @@ -508,16 +514,20 @@ static int __fbnic_pm_resume(struct device *dev) if (fbnic_init_failure(fbd)) return 0; + rtnl_lock(); + netdev_lock(netdev); + fbn = netdev_priv(netdev); /* Reset the queues if needed */ fbnic_reset_queues(fbn, fbn->num_tx_queues, fbn->num_rx_queues); - rtnl_lock(); - netdev_lock(netdev); - - if (netif_running(netdev)) + if (netif_running(netdev)) { err = __fbnic_open(fbn); + /* On failure the vectors are freed, so drop the count */ + if (err) + fbn->num_napi = 0; + } netdev_unlock(netdev); rtnl_unlock(); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c index e7918d3f6aba..10caacffee0f 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -1622,7 +1623,7 @@ fbnic_alloc_qt_page_pools(struct fbnic_net *fbn, struct fbnic_q_triad *qt, return 0; err_destroy_sub0: - page_pool_destroy(pp); + page_pool_destroy(qt->sub0.page_pool); return PTR_ERR(pp); } @@ -1791,7 +1792,7 @@ int fbnic_alloc_napi_vectors(struct fbnic_net *fbn) int err; /* Allocate 1 Tx queue per napi vector */ - if (num_napi < FBNIC_MAX_TXQS && num_napi == num_tx + num_rx) { + if (num_napi <= FBNIC_MAX_TXQS && num_napi == num_tx + num_rx) { while (num_tx) { err = fbnic_alloc_napi_vector(fbd, fbn, num_napi, v_idx, @@ -2853,6 +2854,17 @@ void fbnic_napi_depletion_check(struct net_device *netdev) fbnic_wrfl(fbd); } +/* Returns the napi vector servicing an Rx queue, or NULL if the datapath + * is torn down. The association is published by fbnic_set_netif_napi() + * and cleared by fbnic_reset_netif_napi(), both under the instance lock. + */ +static struct fbnic_napi_vector *fbnic_rxq_nv(struct net_device *dev, int idx) +{ + struct napi_struct *napi = __netif_get_rx_queue(dev, idx)->napi; + + return napi ? container_of(napi, struct fbnic_napi_vector, napi) : NULL; +} + static int fbnic_queue_mem_alloc(struct net_device *dev, struct netdev_queue_config *qcfg, void *qmem, int idx) @@ -2865,8 +2877,16 @@ static int fbnic_queue_mem_alloc(struct net_device *dev, if (!netif_running(dev)) return fbnic_alloc_qt_page_pools(fbn, qt, idx); + /* A failed PCIe recovery or resume can leave the datapath torn down + * while netif_running() is still true. This ndo runs before + * netdev_rx_queue_restart() checks netif_running(), so bail out + * rather than touching rings and vectors that are already freed. + */ + nv = fbnic_rxq_nv(dev, idx); + if (!nv) + return -ENETDOWN; + real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl); - nv = fbn->napi[idx % fbn->num_napi]; fbnic_ring_init(&qt->sub0, real->sub0.doorbell, real->sub0.q_idx, real->sub0.flags); @@ -2917,7 +2937,7 @@ static int fbnic_queue_start(struct net_device *dev, struct fbnic_q_triad *real; real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl); - nv = fbn->napi[idx % fbn->num_napi]; + nv = fbnic_rxq_nv(dev, idx); fbnic_aggregate_ring_bdq_counters(fbn, &real->sub0); fbnic_aggregate_ring_bdq_counters(fbn, &real->sub1); @@ -2939,7 +2959,7 @@ static int fbnic_queue_stop(struct net_device *dev, void *qmem, int idx) int err; real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl); - nv = fbn->napi[idx % fbn->num_napi]; + nv = fbnic_rxq_nv(dev, idx); fbnic_dbg_nv_exit(nv); napi_disable_locked(&nv->napi); diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_main.c b/drivers/net/ethernet/microchip/sparx5/sparx5_main.c index dad713e9ddd5..b530fb6d3d0c 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_main.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_main.c @@ -898,6 +898,7 @@ static int mchp_sparx5_probe(struct platform_device *pdev) dev_info(sparx5->dev, "MAC addr was not set, use random MAC\n"); eth_random_addr(sparx5->base_mac); sparx5->base_mac[5] = 0; + sparx5->base_mac_assign_type = NET_ADDR_RANDOM; } sparx5->fdma_irq = platform_get_irq_byname(sparx5->pdev, "fdma"); diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_main.h b/drivers/net/ethernet/microchip/sparx5/sparx5_main.h index eb57b86fbe22..0f7d8440f614 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_main.h +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_main.h @@ -392,6 +392,7 @@ struct sparx5 { struct notifier_block switchdev_blocking_nb; /* Switch state */ u8 base_mac[ETH_ALEN]; + u8 base_mac_assign_type; /* Associated bridge device (when bridged) */ struct net_device *hw_bridge_dev; /* Bridged interfaces */ diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_netdev.c b/drivers/net/ethernet/microchip/sparx5/sparx5_netdev.c index 4dedf5d92641..b3725cf2dd44 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_netdev.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_netdev.c @@ -294,6 +294,7 @@ struct net_device *sparx5_create_netdev(struct sparx5 *sparx5, u32 portno) ndev->ethtool_ops = &sparx5_ethtool_ops; eth_hw_addr_gen(ndev, sparx5->base_mac, portno + 1); + ndev->addr_assign_type = sparx5->base_mac_assign_type; return ndev; } diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c b/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c index a16ec8136d6d..5953e04ef14e 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c @@ -673,6 +673,9 @@ void sparx5_ptp_deinit(struct sparx5 *sparx5) struct sparx5_port *port; int i; + if (!sparx5->ptp) + return; + if (sparx5->ptp_irq >= 0) { disable_irq(sparx5->ptp_irq); sparx5->ptp_irq = -ENXIO; diff --git a/drivers/net/ethernet/microchip/vcap/vcap_api.c b/drivers/net/ethernet/microchip/vcap/vcap_api.c index 788c0728d763..0f8d98f2094d 100644 --- a/drivers/net/ethernet/microchip/vcap/vcap_api.c +++ b/drivers/net/ethernet/microchip/vcap/vcap_api.c @@ -3527,8 +3527,8 @@ int vcap_filter_rule_keys(struct vcap_rule *rule, if (key == keylist[idx]) { list_del(&ckf->ctrl.list); kfree(ckf); - idx++; err = 0; + break; } } } diff --git a/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c b/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c index 9e7c285eaa6b..960d7513aa8d 100644 --- a/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c +++ b/drivers/net/ethernet/netronome/nfp/crypto/ipsec.c @@ -625,11 +625,12 @@ int nfp_net_ipsec_rx(struct nfp_meta_parsed *meta, struct sk_buff *skb) xa_lock(&nn->xa_ipsec); x = xa_load(&nn->xa_ipsec, saidx); + if (x) + xfrm_state_hold(x); xa_unlock(&nn->xa_ipsec); if (!x) return -EINVAL; - xfrm_state_hold(x); sp->xvec[sp->len++] = x; sp->olen++; xo = xfrm_offload(skb); diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c index ec4fc21fa21f..ee1bc9bd0447 100644 --- a/drivers/net/ethernet/realtek/r8169_main.c +++ b/drivers/net/ethernet/realtek/r8169_main.c @@ -5484,6 +5484,12 @@ static int r8169_mdio_register(struct rtl8169_private *tp) phy_disable_eee_mode(tp->phydev, ETHTOOL_LINK_MODE_2500baseT_Full_BIT); + /* Keep EEE off but the version stays EEE-capable so that link-up + * still clears the MAC TX-LPI bits. + */ + if (tp->mac_version == RTL_GIGA_MAC_VER_46) + phy_disable_eee(tp->phydev); + /* PHY will be woken up in rtl_open() */ phy_suspend(tp->phydev); diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c index 3806cd3dd7f4..953f1d90b9b1 100644 --- a/drivers/net/ethernet/sfc/efx.c +++ b/drivers/net/ethernet/sfc/efx.c @@ -904,6 +904,10 @@ static const struct pci_device_id efx_pci_table[] = { .driver_data = (unsigned long)&efx_x4_nic_type}, {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x2c03), /* X4 PF (FF only) */ .driver_data = (unsigned long)&efx_x4_nic_type}, + {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x8c03), /* X4D PF (FF/LL) */ + .driver_data = (unsigned long)&efx_x4_nic_type}, + {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0xac03), /* X4D PF (FF only) */ + .driver_data = (unsigned long)&efx_x4_nic_type}, {0} /* end of list */ }; diff --git a/drivers/net/ethernet/spacemit/k1_emac.c b/drivers/net/ethernet/spacemit/k1_emac.c index f7f16397a2c2..d641ac26a1e8 100644 --- a/drivers/net/ethernet/spacemit/k1_emac.c +++ b/drivers/net/ethernet/spacemit/k1_emac.c @@ -803,6 +803,9 @@ static void emac_tx_mem_map(struct emac_priv *priv, struct sk_buff *skb) while (i != head) { emac_free_tx_buf(priv, i); + tx_desc_addr = &((struct emac_desc *)tx_ring->desc_addr)[i]; + memset(tx_desc_addr, 0, sizeof(*tx_desc_addr)); + if (++i == tx_ring->total_cnt) i = 0; } diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c index 8d7042e68926..72bdbcb5e863 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c @@ -1162,8 +1162,11 @@ static int gmac_clk_enable(struct rk_priv_data *bsp_priv, bool enable) return ret; ret = clk_prepare_enable(bsp_priv->clk_phy); - if (ret) + if (ret) { + clk_bulk_disable_unprepare(bsp_priv->num_clks, + bsp_priv->clks); return ret; + } rk_configure_io_clksel(bsp_priv); rk_ungate_rmii_clock(bsp_priv); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c index 2994df41ec2c..c6a8f8d73501 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c @@ -474,11 +474,11 @@ static void dwmac4_set_sarc(struct dma_desc *p, u32 sarc_type) sarc_type)); } -static int set_16kib_bfsize(int mtu) +static int set_16kib_bfsize(int len) { int ret = 0; - if (unlikely(mtu >= BUF_SIZE_8KiB)) + if (unlikely(len > BUF_SIZE_8KiB)) ret = BUF_SIZE_16KiB; return ret; } diff --git a/drivers/net/ethernet/stmicro/stmmac/hwif.h b/drivers/net/ethernet/stmicro/stmmac/hwif.h index 9314bcb85c22..857f7562c6c6 100644 --- a/drivers/net/ethernet/stmicro/stmmac/hwif.h +++ b/drivers/net/ethernet/stmicro/stmmac/hwif.h @@ -540,7 +540,7 @@ struct stmmac_mode_ops { bool (*is_jumbo_frm)(unsigned int len, bool enh_desc); int (*jumbo_frm)(struct stmmac_tx_queue *tx_q, struct sk_buff *skb, int csum); - int (*set_16kib_bfsize)(int mtu); + int (*set_16kib_bfsize)(int len); void (*init_desc3)(struct dma_desc *p); void (*refill_desc3)(struct stmmac_rx_queue *rx_q, struct dma_desc *p); void (*clean_desc3)(struct stmmac_tx_queue *tx_q, struct dma_desc *p); diff --git a/drivers/net/ethernet/stmicro/stmmac/ring_mode.c b/drivers/net/ethernet/stmicro/stmmac/ring_mode.c index f7949419eb9f..d2f0c321661d 100644 --- a/drivers/net/ethernet/stmicro/stmmac/ring_mode.c +++ b/drivers/net/ethernet/stmicro/stmmac/ring_mode.c @@ -124,10 +124,10 @@ static void clean_desc3(struct stmmac_tx_queue *tx_q, struct dma_desc *p) p->des3 = 0; } -static int set_16kib_bfsize(int mtu) +static int set_16kib_bfsize(int len) { int ret = 0; - if (unlikely(mtu > BUF_SIZE_8KiB)) + if (unlikely(len > BUF_SIZE_8KiB)) ret = BUF_SIZE_16KiB; return ret; } diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 1fb5f804ea23..ec62fa7418f4 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -1536,17 +1536,17 @@ static unsigned int stmmac_rx_offset(struct stmmac_priv *priv) return NET_SKB_PAD + NET_IP_ALIGN; } -static int stmmac_set_bfsize(int mtu) +static int stmmac_set_bfsize(int len) { int ret; - if (mtu >= BUF_SIZE_8KiB) + if (len > BUF_SIZE_8KiB) ret = BUF_SIZE_16KiB; - else if (mtu >= BUF_SIZE_4KiB) + else if (len > BUF_SIZE_4KiB) ret = BUF_SIZE_8KiB; - else if (mtu >= BUF_SIZE_2KiB) + else if (len > BUF_SIZE_2KiB) ret = BUF_SIZE_4KiB; - else if (mtu > DEFAULT_BUFSIZE) + else if (len > DEFAULT_BUFSIZE) ret = BUF_SIZE_2KiB; else ret = DEFAULT_BUFSIZE; @@ -1629,6 +1629,16 @@ static void stmmac_clear_descriptors(struct stmmac_priv *priv, stmmac_clear_tx_descriptors(priv, dma_conf, queue); } +static bool stmmac_rx_check_buf2_cap(struct stmmac_priv *priv) +{ + /* Only cores that can program an independent secondary RX buffer + * (used for scatter-gather overflow or split-header payload) back + * buffer2. Legacy cores have no set_sec_addr op, so buffer2 is + * never handed to the hardware there. + */ + return priv->hw->desc && priv->hw->desc->set_sec_addr; +} + /** * stmmac_init_rx_buffers - init the RX descriptor buffer. * @priv: driver private structure @@ -1659,16 +1669,13 @@ static int stmmac_init_rx_buffers(struct stmmac_priv *priv, buf->page_offset = stmmac_rx_offset(priv); } - if (priv->sph_active && !buf->sec_page) { + if (stmmac_rx_check_buf2_cap(priv) && !buf->sec_page) { buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); if (!buf->sec_page) return -ENOMEM; buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); - } else { - buf->sec_page = NULL; - stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); } buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; @@ -2256,13 +2263,8 @@ static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv, pp_params.nid = dev_to_node(priv->device); pp_params.dev = priv->device; pp_params.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; - pp_params.offset = stmmac_rx_offset(priv); - pp_params.max_len = dma_conf->dma_buf_sz; - - if (priv->sph_active) { - pp_params.offset = 0; - pp_params.max_len += stmmac_rx_offset(priv); - } + pp_params.offset = 0; + pp_params.max_len = dma_conf->dma_buf_sz + stmmac_rx_offset(priv); rx_q->page_pool = page_pool_create(&pp_params); if (IS_ERR(rx_q->page_pool)) { @@ -4063,7 +4065,7 @@ static struct stmmac_dma_conf * stmmac_setup_dma_desc(struct stmmac_priv *priv, unsigned int mtu) { struct stmmac_dma_conf *dma_conf; - int bfsize, ret; + int bfsize, len, ret; u8 chan; dma_conf = kzalloc_obj(*dma_conf); @@ -4073,13 +4075,15 @@ stmmac_setup_dma_desc(struct stmmac_priv *priv, unsigned int mtu) return ERR_PTR(-ENOMEM); } - /* Returns 0 or BUF_SIZE_16KiB if mtu > 8KiB and dwmac4 or ring mode */ - bfsize = stmmac_set_16kib_bfsize(priv, mtu); + len = mtu + ETH_HLEN + 2 * VLAN_HLEN + ETH_FCS_LEN; + + /* Returns 0 or BUF_SIZE_16KiB if len > 8KiB and dwmac4 or ring mode */ + bfsize = stmmac_set_16kib_bfsize(priv, len); if (bfsize < 0) bfsize = 0; if (bfsize < BUF_SIZE_16KiB) - bfsize = stmmac_set_bfsize(mtu); + bfsize = stmmac_set_bfsize(len); dma_conf->dma_buf_sz = bfsize; /* Chose the tx/rx size from the already defined one in the @@ -5097,7 +5101,7 @@ static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) break; } - if (priv->sph_active && !buf->sec_page) { + if (stmmac_rx_check_buf2_cap(priv) && !buf->sec_page) { buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); if (!buf->sec_page) break; @@ -5108,10 +5112,8 @@ static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; stmmac_set_desc_addr(priv, p, buf->addr); - if (priv->sph_active) - stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); - else - stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); + stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, + stmmac_rx_check_buf2_cap(priv)); stmmac_refill_desc3(priv, rx_q, p); rx_q->rx_count_frames++; @@ -5142,7 +5144,9 @@ static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv, unsigned int plen = 0, hlen = 0; int coe = priv->hw->rx_csum; - /* Not first descriptor, buffer is always zero */ + /* Not first descriptor, SPH enabled: buffer1 only carries the + * split header of the first descriptor, so it is zero here. + */ if (priv->sph_active && len) return 0; @@ -5153,14 +5157,16 @@ static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv, return hlen; } - /* First descriptor, not last descriptor and not split header */ + /* Not last descriptor and not split header: buffer1 is fully filled */ if (status & rx_not_ls) return priv->dma_conf.dma_buf_sz; plen = stmmac_get_rx_frame_len(priv, p, coe); - /* First descriptor and last descriptor and not split header */ - return min_t(unsigned int, priv->dma_conf.dma_buf_sz, plen); + /* Last descriptor and not split header: buffer1 holds the remaining + * bytes of the frame, up to dma_buf_sz + */ + return min_t(unsigned int, priv->dma_conf.dma_buf_sz, plen - len); } static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv, @@ -5170,8 +5176,7 @@ static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv, int coe = priv->hw->rx_csum; unsigned int plen = 0; - /* Not split header, buffer is not available */ - if (!priv->sph_active) + if (!stmmac_rx_check_buf2_cap(priv)) return 0; /* For GMAC4, when split header is enabled, in some rare cases, the @@ -5188,14 +5193,15 @@ static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv, * Thus 'plen - len' always gives the correct length of buf2. */ - /* Not GMAC4 and not last descriptor */ - if (priv->plat->core_type != DWMAC_CORE_GMAC4 && (status & rx_not_ls)) + /* Not GMAC4, or non-SPH and not last descriptor */ + if ((priv->plat->core_type != DWMAC_CORE_GMAC4 || !priv->sph_active) && + (status & rx_not_ls)) return priv->dma_conf.dma_buf_sz; /* GMAC4 or last descriptor */ plen = stmmac_get_rx_frame_len(priv, p, coe); - return plen - len; + return plen > len ? plen - len : 0; } static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue, @@ -5718,6 +5724,7 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) unsigned int desc_size; struct sk_buff *skb = NULL; struct stmmac_xdp_buff ctx; + bool fcs_stripped = false; int xdp_status = 0; int bufsz; @@ -5807,24 +5814,25 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) len += buf2_len; /* ACS is disabled; strip manually. */ - if (likely(!(status & rx_not_ls))) { - if (buf2_len) { - buf2_len -= ETH_FCS_LEN; - len -= ETH_FCS_LEN; - } else if (buf1_len) { - buf1_len -= ETH_FCS_LEN; - len -= ETH_FCS_LEN; - } - } + if (likely(!(status & rx_not_ls))) + len -= ETH_FCS_LEN; if (!skb) { unsigned int pre_len, sync_len; + /* Each frame starts here: reset the FCS handling */ + fcs_stripped = false; + dma_sync_single_for_cpu(priv->device, buf->addr, buf1_len, dma_dir); net_prefetch(page_address(buf->page) + buf->page_offset); + if (stmmac_xdp_is_enabled(priv) && !buf2_len) { + buf1_len -= ETH_FCS_LEN; + fcs_stripped = true; + } + xdp_init_buff(&ctx.xdp, bufsz, &rx_q->xdp_rxq); xdp_prepare_buff(&ctx.xdp, page_address(buf->page), buf->page_offset, buf1_len, true); @@ -5887,6 +5895,7 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) if (!skb) { page_pool_recycle_direct(rx_q->page_pool, buf->page); + buf->page = NULL; rx_dropped++; count++; goto drain_data; @@ -5924,6 +5933,15 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) /* Got entire packet into SKB. Finish it. */ + /* Remove FCS if needed */ + if (!fcs_stripped && pskb_trim(skb, len)) { + dev_kfree_skb_any(skb); + skb = NULL; + rx_dropped++; + count++; + continue; + } + stmmac_get_rx_hwtstamp(priv, p, np, skb); if (priv->hw->hw_vlan_en) diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c index 6372ec7c3f31..c25dc9f89270 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +30,7 @@ struct stmmachdr { sizeof(struct stmmachdr)) #define STMMAC_TEST_PKT_MAGIC 0xdeadcafecafedeadULL #define STMMAC_LB_TIMEOUT msecs_to_jiffies(200) +#define STMMAC_SFT_MAX_LPI (5 * USEC_PER_SEC) struct stmmac_packet_attrs { int vlan; @@ -237,6 +239,10 @@ struct stmmac_test_priv { struct stmmac_packet_attrs *packet; struct packet_type pt; struct completion comp; + __be16 packet_type; + int (*func)(struct sk_buff *skb, struct net_device *ndev, + struct packet_type *pt, struct net_device *orig_ndev); + bool capture_all; int double_vlan; int vlan_id; int ok; @@ -316,6 +322,52 @@ static int stmmac_test_loopback_validate(struct sk_buff *skb, return 0; } +static int stmmac_sft_filter(struct sk_buff *skb, struct net_device *ndev, + struct packet_type *pt, + struct net_device *orig_ndev) +{ + struct stmmac_test_priv *tpriv = pt->af_packet_priv; + struct ethhdr *hdr = eth_hdr(skb); + int ret = 0; + + if (hdr->h_proto == tpriv->packet_type) { + struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC); + + if (nskb) + ret = tpriv->func(nskb, ndev, pt, orig_ndev); + } + + kfree_skb(skb); + return ret; +} + +static void stmmac_sft_add_pack(struct packet_type *pt) +{ + struct stmmac_test_priv *tpriv = pt->af_packet_priv; + + if (netdev_uses_dsa(tpriv->pt.dev) || tpriv->capture_all) { + tpriv->packet_type = tpriv->pt.type; + tpriv->func = tpriv->pt.func; + + /* DSA conduit will report ETH_P_XDSA, so our packet handler + * won't match. Let's register a ETH_P_ALL match and filter + * manually in stmmac_sft_filter. This is also useful for + * VLAN tests, to capture packets otherwise marked as + * OTHERHOST. + */ + tpriv->pt.type = htons(ETH_P_ALL); + tpriv->pt.func = stmmac_sft_filter; + tpriv->pt.ignore_outgoing = true; + } + + dev_add_pack(pt); +} + +static void stmmac_sft_remove_pack(struct packet_type *pt) +{ + dev_remove_pack(pt); +} + static int __stmmac_test_loopback(struct stmmac_priv *priv, struct stmmac_packet_attrs *attr) { @@ -337,7 +389,7 @@ static int __stmmac_test_loopback(struct stmmac_priv *priv, tpriv->packet = attr; if (!attr->dont_wait) - dev_add_pack(&tpriv->pt); + stmmac_sft_add_pack(&tpriv->pt); skb = stmmac_test_get_udp_skb(priv, attr); if (!skb) { @@ -360,7 +412,7 @@ static int __stmmac_test_loopback(struct stmmac_priv *priv, cleanup: if (!attr->dont_wait) - dev_remove_pack(&tpriv->pt); + stmmac_sft_remove_pack(&tpriv->pt); kfree(tpriv); return ret; } @@ -414,12 +466,16 @@ static int stmmac_test_mmc(struct stmmac_priv *priv) static int stmmac_test_eee(struct stmmac_priv *priv) { struct stmmac_extra_stats *initial, *final; - int retries = 10; + unsigned long timeout, max_duration; int ret; if (!priv->dma_cap.eee || !priv->eee_active) return -EOPNOTSUPP; + /* Bail out if the configured LPI timer is too long */ + if (priv->tx_lpi_timer > STMMAC_SFT_MAX_LPI) + return -EOPNOTSUPP; + initial = kzalloc_obj(*initial); if (!initial) return -ENOMEM; @@ -430,14 +486,21 @@ static int stmmac_test_eee(struct stmmac_priv *priv) goto out_free_initial; } + /* Snapshot stats, we want to count the in_lpi events. We may enter + * LPI just after the packet was sent. + */ memcpy(initial, &priv->xstats, sizeof(*initial)); + /* Send a frame, then wait to enter LPI */ ret = stmmac_test_mac_loopback(priv); if (ret) goto out_free_final; + max_duration = usecs_to_jiffies(2 * priv->tx_lpi_timer); + /* We have no traffic in the line so, sooner or later it will go LPI */ - while (--retries) { + timeout = jiffies + max_duration; + while (!time_after(jiffies, timeout)) { memcpy(final, &priv->xstats, sizeof(*final)); if (final->irq_tx_path_in_lpi_mode_n > @@ -446,20 +509,38 @@ static int stmmac_test_eee(struct stmmac_priv *priv) msleep(100); } - if (!retries) { + memcpy(final, &priv->xstats, sizeof(*final)); + if (final->irq_tx_path_in_lpi_mode_n <= + initial->irq_tx_path_in_lpi_mode_n) { ret = -ETIMEDOUT; goto out_free_final; } - if (final->irq_tx_path_in_lpi_mode_n <= - initial->irq_tx_path_in_lpi_mode_n) { - ret = -EINVAL; + /* Re-snapshot, as we want to measure exit_lpi events. We should be + * in LPI right now. + */ + memcpy(initial, &priv->xstats, sizeof(*initial)); + + /* TX something so we go out of LPI */ + ret = stmmac_test_mac_loopback(priv); + if (ret) goto out_free_final; + + /* Wait for the exit LPI interrupt */ + timeout = jiffies + max_duration; + while (!time_after(jiffies, timeout)) { + memcpy(final, &priv->xstats, sizeof(*final)); + + if (final->irq_tx_path_exit_lpi_mode_n > + initial->irq_tx_path_exit_lpi_mode_n) + break; + msleep(100); } + memcpy(final, &priv->xstats, sizeof(*final)); if (final->irq_tx_path_exit_lpi_mode_n <= initial->irq_tx_path_exit_lpi_mode_n) { - ret = -EINVAL; + ret = -ETIMEDOUT; goto out_free_final; } @@ -767,7 +848,7 @@ static int stmmac_test_flowctrl(struct stmmac_priv *priv) tpriv->pt.func = stmmac_test_flowctrl_validate; tpriv->pt.dev = priv->dev; tpriv->pt.af_packet_priv = tpriv; - dev_add_pack(&tpriv->pt); + stmmac_sft_add_pack(&tpriv->pt); /* Compute minimum number of packets to make FIFO full */ pkt_count = rx_fifo_size; @@ -823,7 +904,7 @@ static int stmmac_test_flowctrl(struct stmmac_priv *priv) cleanup: dev_mc_del(priv->dev, paddr); dev_set_promiscuity(priv->dev, -1); - dev_remove_pack(&tpriv->pt); + stmmac_sft_remove_pack(&tpriv->pt); kfree(tpriv); return ret; } @@ -865,6 +946,11 @@ static int stmmac_test_vlan_validate(struct sk_buff *skb, goto out; if (skb_headlen(skb) < (STMMAC_TEST_PKT_SIZE - ETH_HLEN)) goto out; + + ehdr = (struct ethhdr *)skb_mac_header(skb); + if (!ether_addr_equal_unaligned(ehdr->h_dest, tpriv->packet->dst)) + goto out; + if (tpriv->vlan_id) { if (skb->vlan_proto != htons(proto)) goto out; @@ -876,10 +962,6 @@ static int stmmac_test_vlan_validate(struct sk_buff *skb, } } - ehdr = (struct ethhdr *)skb_mac_header(skb); - if (!ether_addr_equal_unaligned(ehdr->h_dest, tpriv->packet->dst)) - goto out; - ihdr = ip_hdr(skb); if (tpriv->double_vlan) ihdr = (struct iphdr *)(skb_network_header(skb) + 4); @@ -921,6 +1003,7 @@ static int __stmmac_test_vlanfilt(struct stmmac_priv *priv) tpriv->pt.dev = priv->dev; tpriv->pt.af_packet_priv = tpriv; tpriv->packet = &attr; + tpriv->capture_all = true; /* * As we use HASH filtering, false positives may appear. This is a @@ -928,18 +1011,20 @@ static int __stmmac_test_vlanfilt(struct stmmac_priv *priv) * HASH values. */ tpriv->vlan_id = 0x123; - dev_add_pack(&tpriv->pt); ret = vlan_vid_add(priv->dev, htons(ETH_P_8021Q), tpriv->vlan_id); if (ret) goto cleanup; + attr.vlan = 1; + attr.dst = priv->dev->dev_addr; + attr.sport = 9; + attr.dport = 9; + + stmmac_sft_add_pack(&tpriv->pt); + for (i = 0; i < 4; i++) { - attr.vlan = 1; attr.vlan_id_out = tpriv->vlan_id + i; - attr.dst = priv->dev->dev_addr; - attr.sport = 9; - attr.dport = 9; skb = stmmac_test_get_udp_skb(priv, &attr); if (!skb) { @@ -966,9 +1051,9 @@ static int __stmmac_test_vlanfilt(struct stmmac_priv *priv) } vlan_del: + stmmac_sft_remove_pack(&tpriv->pt); vlan_vid_del(priv->dev, htons(ETH_P_8021Q), tpriv->vlan_id); cleanup: - dev_remove_pack(&tpriv->pt); kfree(tpriv); return ret; } @@ -1015,6 +1100,7 @@ static int __stmmac_test_dvlanfilt(struct stmmac_priv *priv) tpriv->pt.dev = priv->dev; tpriv->pt.af_packet_priv = tpriv; tpriv->packet = &attr; + tpriv->capture_all = true; /* * As we use HASH filtering, false positives may appear. This is a @@ -1022,18 +1108,20 @@ static int __stmmac_test_dvlanfilt(struct stmmac_priv *priv) * HASH values. */ tpriv->vlan_id = 0x123; - dev_add_pack(&tpriv->pt); ret = vlan_vid_add(priv->dev, htons(ETH_P_8021AD), tpriv->vlan_id); if (ret) goto cleanup; + attr.vlan = 2; + attr.dst = priv->dev->dev_addr; + attr.sport = 9; + attr.dport = 9; + + stmmac_sft_add_pack(&tpriv->pt); + for (i = 0; i < 4; i++) { - attr.vlan = 2; attr.vlan_id_out = tpriv->vlan_id + i; - attr.dst = priv->dev->dev_addr; - attr.sport = 9; - attr.dport = 9; skb = stmmac_test_get_udp_skb(priv, &attr); if (!skb) { @@ -1060,9 +1148,9 @@ static int __stmmac_test_dvlanfilt(struct stmmac_priv *priv) } vlan_del: + stmmac_sft_remove_pack(&tpriv->pt); vlan_vid_del(priv->dev, htons(ETH_P_8021AD), tpriv->vlan_id); cleanup: - dev_remove_pack(&tpriv->pt); kfree(tpriv); return ret; } @@ -1293,7 +1381,7 @@ static int stmmac_test_vlanoff_common(struct stmmac_priv *priv, bool svlan) tpriv->pt.af_packet_priv = tpriv; tpriv->packet = &attr; tpriv->vlan_id = 0x123; - dev_add_pack(&tpriv->pt); + tpriv->capture_all = true; ret = vlan_vid_add(priv->dev, htons(proto), tpriv->vlan_id); if (ret) @@ -1301,6 +1389,8 @@ static int stmmac_test_vlanoff_common(struct stmmac_priv *priv, bool svlan) attr.dst = priv->dev->dev_addr; + stmmac_sft_add_pack(&tpriv->pt); + skb = stmmac_test_get_udp_skb(priv, &attr); if (!skb) { ret = -ENOMEM; @@ -1318,9 +1408,9 @@ static int stmmac_test_vlanoff_common(struct stmmac_priv *priv, bool svlan) ret = tpriv->ok ? 0 : -ETIMEDOUT; vlan_del: + stmmac_sft_remove_pack(&tpriv->pt); vlan_vid_del(priv->dev, htons(proto), tpriv->vlan_id); cleanup: - dev_remove_pack(&tpriv->pt); kfree(tpriv); return ret; } @@ -1332,7 +1422,7 @@ static int stmmac_test_vlanoff(struct stmmac_priv *priv) static int stmmac_test_svlanoff(struct stmmac_priv *priv) { - if (!priv->dma_cap.dvlan) + if (!(priv->dev->features & NETIF_F_HW_VLAN_STAG_TX)) return -EOPNOTSUPP; return stmmac_test_vlanoff_common(priv, true); } @@ -1699,6 +1789,9 @@ static int __stmmac_test_jumbo(struct stmmac_priv *priv, u16 queue) struct stmmac_packet_attrs attr = { }; int size = priv->dma_conf.dma_buf_sz; + if (!dwmac_is_xmac(priv->plat->core_type)) + size -= NET_IP_ALIGN; + attr.dst = priv->dev->dev_addr; attr.max_size = size - ETH_FCS_LEN; attr.queue_mapping = queue; diff --git a/drivers/net/ethernet/ti/netcp_core.c b/drivers/net/ethernet/ti/netcp_core.c index eb8fc2ed05f4..4f9e20468bbb 100644 --- a/drivers/net/ethernet/ti/netcp_core.c +++ b/drivers/net/ethernet/ti/netcp_core.c @@ -2225,7 +2225,7 @@ static int netcp_probe(struct platform_device *pdev) return -ENOMEM; pm_runtime_enable(&pdev->dev); - ret = pm_runtime_get_sync(&pdev->dev); + ret = pm_runtime_resume_and_get(&pdev->dev); if (ret < 0) { dev_err(dev, "Failed to enable NETCP power-domain\n"); pm_runtime_disable(&pdev->dev); diff --git a/drivers/net/ethernet/wangxun/libwx/wx_hw.c b/drivers/net/ethernet/wangxun/libwx/wx_hw.c index 122c4952d203..113552586be7 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_hw.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_hw.c @@ -2518,6 +2518,7 @@ int wx_sw_init(struct wx *wx) } spin_lock_init(&wx->hw_stats_lock); + spin_lock_init(&wx->ptp_tx_lock); mutex_init(&wx->reset_lock); bitmap_zero(wx->state, WX_STATE_NBITS); bitmap_zero(wx->flags, WX_PF_FLAGS_NBITS); diff --git a/drivers/net/ethernet/wangxun/libwx/wx_lib.c b/drivers/net/ethernet/wangxun/libwx/wx_lib.c index ed5aad7857bd..dcbf5811046e 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_lib.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_lib.c @@ -1200,9 +1200,11 @@ static int wx_tx_map(struct wx_ring *tx_ring, i--; } - dev_kfree_skb_any(first->skb); - first->skb = NULL; - + /* first->skb is released by the caller, which keeps a reference on it + * until the PTP cleanup has compared it against wx->ptp_tx_skb. That + * prevents the address from being reused by a newer request while the + * comparison is pending. + */ tx_ring->next_to_use = i; return -ENOMEM; @@ -1649,9 +1651,11 @@ static netdev_tx_t wx_xmit_frame_ring(struct sk_buff *skb, if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && wx->ptp_clock) { + unsigned long flags; + + spin_lock_irqsave(&wx->ptp_tx_lock, flags); if (wx->tstamp_config.tx_type == HWTSTAMP_TX_ON && - !test_and_set_bit_lock(WX_STATE_PTP_TX_IN_PROGRESS, - wx->state)) { + !test_and_set_bit(WX_STATE_PTP_TX_IN_PROGRESS, wx->state)) { skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; tx_flags |= WX_TX_FLAGS_TSTAMP; wx->ptp_tx_skb = skb_get(skb); @@ -1659,6 +1663,7 @@ static netdev_tx_t wx_xmit_frame_ring(struct sk_buff *skb, } else { wx->tx_hwtstamp_skipped++; } + spin_unlock_irqrestore(&wx->ptp_tx_lock, flags); } /* record initial flags and protocol */ @@ -1677,19 +1682,35 @@ static netdev_tx_t wx_xmit_frame_ring(struct sk_buff *skb, wx->atr(tx_ring, first, ptype); if (wx_tx_map(tx_ring, first, hdr_len)) - goto cleanup_tx_tstamp; + goto out_drop; return NETDEV_TX_OK; out_drop: + /* The frame never reached the hardware, so no timestamp will ever be + * reported for it and the request has to be cancelled. The slot is + * shared, though: wx_ptp_clear_tx_timestamp() or wx_ptp_tx_hang() may + * have dropped our request already, and a transmit on another queue + * can have claimed the slot since. Only cancel it while it is still + * ours, otherwise we would free somebody else's skb and release their + * in-progress bit. + */ + if (unlikely(tx_flags & WX_TX_FLAGS_TSTAMP)) { + struct sk_buff *ptp_tx_skb = NULL; + unsigned long flags; + + spin_lock_irqsave(&wx->ptp_tx_lock, flags); + if (wx->ptp_tx_skb == skb) { + ptp_tx_skb = wx->ptp_tx_skb; + wx->ptp_tx_skb = NULL; + clear_bit(WX_STATE_PTP_TX_IN_PROGRESS, wx->state); + wx->tx_hwtstamp_errors++; + } + spin_unlock_irqrestore(&wx->ptp_tx_lock, flags); + + dev_kfree_skb_any(ptp_tx_skb); + } dev_kfree_skb_any(first->skb); first->skb = NULL; -cleanup_tx_tstamp: - if (unlikely(tx_flags & WX_TX_FLAGS_TSTAMP)) { - dev_kfree_skb_any(wx->ptp_tx_skb); - wx->ptp_tx_skb = NULL; - wx->tx_hwtstamp_errors++; - clear_bit_unlock(WX_STATE_PTP_TX_IN_PROGRESS, wx->state); - } return NETDEV_TX_OK; } diff --git a/drivers/net/ethernet/wangxun/libwx/wx_ptp.c b/drivers/net/ethernet/wangxun/libwx/wx_ptp.c index 4708e7f3958f..65b8937f6e94 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_ptp.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_ptp.c @@ -129,6 +129,34 @@ static int wx_ptp_settime64(struct ptp_clock_info *ptp, return 0; } +/** + * __wx_ptp_detach_tx_skb - detach the skb tracking the Tx timestamp request + * @wx: the private board structure + * + * Detach the skb of the outstanding request and release the in-progress bit, + * so that a new request can be submitted. + * + * This performs no register access. Callers that need a timestamp the hardware + * may have left latched must unlatch it themselves, while the device is known + * to be alive. wx_ptp_quiesce() runs during PCIe error recovery, where MMIO is + * not reliable, and therefore deliberately skips the unlatch. + * + * Context: Expects wx->ptp_tx_lock to be held by the caller. + * Return: the detached skb, or NULL if no request was outstanding. The caller + * owns the returned reference and must release it once the lock is dropped. + */ +static struct sk_buff *__wx_ptp_detach_tx_skb(struct wx *wx) +{ + struct sk_buff *skb = wx->ptp_tx_skb; + + lockdep_assert_held(&wx->ptp_tx_lock); + + wx->ptp_tx_skb = NULL; + clear_bit(WX_STATE_PTP_TX_IN_PROGRESS, wx->state); + + return skb; +} + /** * wx_ptp_clear_tx_timestamp - utility function to clear Tx timestamp state * @wx: the private board structure @@ -139,12 +167,16 @@ static int wx_ptp_settime64(struct ptp_clock_info *ptp, */ static void wx_ptp_clear_tx_timestamp(struct wx *wx) { + struct sk_buff *skb; + unsigned long flags; + + spin_lock_irqsave(&wx->ptp_tx_lock, flags); + /* Unlatch a timestamp the hardware may have left pending. */ rd32ptp(wx, WX_TSC_1588_STMPH); - if (wx->ptp_tx_skb) { - dev_kfree_skb_any(wx->ptp_tx_skb); - wx->ptp_tx_skb = NULL; - } - clear_bit_unlock(WX_STATE_PTP_TX_IN_PROGRESS, wx->state); + skb = __wx_ptp_detach_tx_skb(wx); + spin_unlock_irqrestore(&wx->ptp_tx_lock, flags); + + dev_kfree_skb_any(skb); } /** @@ -175,49 +207,54 @@ static void wx_ptp_convert_to_hwtstamp(struct wx *wx, } /** - * wx_ptp_tx_hwtstamp - utility function which checks for TX time stamp + * wx_ptp_tx_hwtstamp_work - check for a pending Tx time stamp * @wx: the private board struct * - * if the timestamp is valid, we convert it into the timecounter ns - * value, then store that result into the shhwtstamps structure which - * is passed up the network stack + * If a Tx timestamp request is outstanding and the hardware has latched a + * valid value, we convert it into the timecounter ns value, then store that + * result into the shhwtstamps structure which is passed up the network stack. + * + * Return: 0 when there is nothing left to poll for, -1 when the timestamp is + * not available yet and the caller should poll again. */ -static void wx_ptp_tx_hwtstamp(struct wx *wx) -{ - struct skb_shared_hwtstamps shhwtstamps; - struct sk_buff *skb = wx->ptp_tx_skb; - u64 regval = 0; - - regval |= (u64)rd32ptp(wx, WX_TSC_1588_STMPL); - regval |= (u64)rd32ptp(wx, WX_TSC_1588_STMPH) << 32; - - wx_ptp_convert_to_hwtstamp(wx, &shhwtstamps, regval); - - wx->ptp_tx_skb = NULL; - clear_bit_unlock(WX_STATE_PTP_TX_IN_PROGRESS, wx->state); - skb_tstamp_tx(skb, &shhwtstamps); - dev_kfree_skb_any(skb); - wx->tx_hwtstamp_pkts++; -} - static int wx_ptp_tx_hwtstamp_work(struct wx *wx) { + struct skb_shared_hwtstamps shhwtstamps; + unsigned long flags; + struct sk_buff *skb; u32 tsynctxctl; + u64 regval = 0; + + spin_lock_irqsave(&wx->ptp_tx_lock, flags); /* we have to have a valid skb to poll for a timestamp */ if (!wx->ptp_tx_skb) { - wx_ptp_clear_tx_timestamp(wx); + rd32ptp(wx, WX_TSC_1588_STMPH); + __wx_ptp_detach_tx_skb(wx); + spin_unlock_irqrestore(&wx->ptp_tx_lock, flags); return 0; } /* stop polling once we have a valid timestamp */ tsynctxctl = rd32ptp(wx, WX_TSC_1588_CTL); - if (tsynctxctl & WX_TSC_1588_CTL_VALID) { - wx_ptp_tx_hwtstamp(wx); - return 0; + if (!(tsynctxctl & WX_TSC_1588_CTL_VALID)) { + spin_unlock_irqrestore(&wx->ptp_tx_lock, flags); + return -1; } - return -1; + regval |= (u64)rd32ptp(wx, WX_TSC_1588_STMPL); + regval |= (u64)rd32ptp(wx, WX_TSC_1588_STMPH) << 32; + skb = wx->ptp_tx_skb; + wx->ptp_tx_skb = NULL; + clear_bit(WX_STATE_PTP_TX_IN_PROGRESS, wx->state); + spin_unlock_irqrestore(&wx->ptp_tx_lock, flags); + + wx_ptp_convert_to_hwtstamp(wx, &shhwtstamps, regval); + skb_tstamp_tx(skb, &shhwtstamps); + dev_kfree_skb_any(skb); + wx->tx_hwtstamp_pkts++; + + return 0; } /** @@ -296,24 +333,29 @@ static void wx_ptp_rx_hang(struct wx *wx) */ static void wx_ptp_tx_hang(struct wx *wx) { - bool timeout = time_is_before_jiffies(wx->ptp_tx_start + - WX_PTP_TX_TIMEOUT); + struct sk_buff *skb = NULL; + unsigned long flags; - if (!wx->ptp_tx_skb) - return; - - if (!test_bit(WX_STATE_PTP_TX_IN_PROGRESS, wx->state)) - return; + spin_lock_irqsave(&wx->ptp_tx_lock, flags); /* If we haven't received a timestamp within the timeout, it is * reasonable to assume that it will never occur, so we can unlock the * timestamp bit when this occurs. */ - if (timeout) { - wx_ptp_clear_tx_timestamp(wx); - wx->tx_hwtstamp_timeouts++; - dev_warn(&wx->pdev->dev, "clearing Tx timestamp hang\n"); + if (wx->ptp_tx_skb && + test_bit(WX_STATE_PTP_TX_IN_PROGRESS, wx->state) && + time_is_before_jiffies(wx->ptp_tx_start + WX_PTP_TX_TIMEOUT)) { + rd32ptp(wx, WX_TSC_1588_STMPH); + skb = __wx_ptp_detach_tx_skb(wx); } + spin_unlock_irqrestore(&wx->ptp_tx_lock, flags); + + if (!skb) + return; + + dev_kfree_skb_any(skb); + wx->tx_hwtstamp_timeouts++; + dev_warn(&wx->pdev->dev, "clearing Tx timestamp hang\n"); } static long wx_ptp_do_aux_work(struct ptp_clock_info *ptp) @@ -841,6 +883,9 @@ EXPORT_SYMBOL(wx_ptp_stop); void wx_ptp_quiesce(struct wx *wx) { + struct sk_buff *skb; + unsigned long flags; + if (!test_and_clear_bit(WX_STATE_PTP_RUNNING, wx->state)) return; @@ -849,11 +894,14 @@ void wx_ptp_quiesce(struct wx *wx) if (wx->ptp_clock) ptp_cancel_worker_sync(wx->ptp_clock); - if (wx->ptp_tx_skb) { - dev_kfree_skb_any(wx->ptp_tx_skb); - wx->ptp_tx_skb = NULL; - } - clear_bit_unlock(WX_STATE_PTP_TX_IN_PROGRESS, wx->state); + /* Drop a pending Tx timestamp request. Do not touch the registers + * here: quiesce runs during PCIe error recovery, where the device may + * already be gone and MMIO is not reliable. + */ + spin_lock_irqsave(&wx->ptp_tx_lock, flags); + skb = __wx_ptp_detach_tx_skb(wx); + spin_unlock_irqrestore(&wx->ptp_tx_lock, flags); + dev_kfree_skb_any(skb); if (wx->ptp_clock) { ptp_clock_unregister(wx->ptp_clock); diff --git a/drivers/net/ethernet/wangxun/libwx/wx_type.h b/drivers/net/ethernet/wangxun/libwx/wx_type.h index 9454e90258d8..afd980dbb793 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_type.h +++ b/drivers/net/ethernet/wangxun/libwx/wx_type.h @@ -1430,6 +1430,8 @@ struct wx { unsigned long last_overflow_check; unsigned long last_rx_ptp_check; unsigned long ptp_tx_start; + /* protects ptp_tx_skb, ptp_tx_start and the in-progress state bit */ + spinlock_t ptp_tx_lock; seqlock_t hw_tc_lock; /* seqlock for ptp */ struct cyclecounter hw_cc; struct timecounter hw_tc; diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_fdir.c b/drivers/net/ethernet/wangxun/txgbe/txgbe_fdir.c index a84010828551..59a47532618c 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_fdir.c +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_fdir.c @@ -591,15 +591,24 @@ static void txgbe_fdir_filter_restore(struct wx *wx) queue = TXGBE_RDB_FDIR_DROP_QUEUE; } else { u32 ring = ethtool_get_flow_spec_ring(filter->action); + u8 vf = ethtool_get_flow_spec_ring_vf(filter->action); - if (ring >= wx->num_rx_queues) { + if (!vf && ring >= wx->num_rx_queues) { wx_err(wx, "FDIR restore failed, ring:%u\n", ring); continue; + } else if (vf && (vf > wx->num_vfs || + ring >= wx->num_rx_queues_per_pool)) { + wx_err(wx, "FDIR restore failed, vf:%u, ring:%u\n", + vf, ring); + continue; } /* Map the ring onto the absolute queue index */ - queue = wx->rx_ring[ring]->reg_idx; + if (!vf) + queue = wx->rx_ring[ring]->reg_idx; + else + queue = ((vf - 1) * wx->num_rx_queues_per_pool) + ring; } ret = txgbe_fdir_write_perfect_filter(wx, diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c index 6f9f3aceffaa..78a19b134632 100644 --- a/drivers/net/macsec.c +++ b/drivers/net/macsec.c @@ -3539,6 +3539,22 @@ static int macsec_dev_init(struct net_device *dev) if (err) return err; + err = -ENOMEM; + macsec->stats = netdev_alloc_pcpu_stats(struct pcpu_secy_stats); + if (!macsec->stats) + goto destroy_gro_cells; + + macsec->secy.tx_sc.stats = + netdev_alloc_pcpu_stats(struct pcpu_tx_sc_stats); + if (!macsec->secy.tx_sc.stats) + goto free_secy_stats; + + macsec->secy.tx_sc.md_dst = metadata_dst_alloc(0, METADATA_MACSEC, + GFP_KERNEL); + if (!macsec->secy.tx_sc.md_dst) + goto free_tx_sc_stats; + macsec->secy.tx_sc.md_dst->u.macsec_info.sci = macsec->secy.sci; + macsec_inherit_tso_max(dev); dev->hw_features = real_dev->hw_features & MACSEC_OFFLOAD_FEATURES; @@ -3551,8 +3567,6 @@ static int macsec_dev_init(struct net_device *dev) macsec_set_head_tail_room(dev); - if (is_zero_ether_addr(dev->dev_addr)) - eth_hw_addr_inherit(dev, real_dev); if (is_zero_ether_addr(dev->broadcast)) memcpy(dev->broadcast, real_dev->broadcast, dev->addr_len); @@ -3560,6 +3574,14 @@ static int macsec_dev_init(struct net_device *dev) netdev_hold(real_dev, &macsec->dev_tracker, GFP_KERNEL); return 0; + +free_tx_sc_stats: + free_percpu(macsec->secy.tx_sc.stats); +free_secy_stats: + free_percpu(macsec->stats); +destroy_gro_cells: + gro_cells_destroy(&macsec->gro_cells); + return err; } static void macsec_dev_uninit(struct net_device *dev) @@ -4116,26 +4138,11 @@ static sci_t dev_to_sci(struct net_device *dev, __be16 port) return make_sci(dev->dev_addr, port); } -static int macsec_add_dev(struct net_device *dev, sci_t sci, u8 icv_len) +static void macsec_init_secy(struct net_device *dev, sci_t sci, u8 icv_len) { struct macsec_dev *macsec = macsec_priv(dev); struct macsec_secy *secy = &macsec->secy; - macsec->stats = netdev_alloc_pcpu_stats(struct pcpu_secy_stats); - if (!macsec->stats) - return -ENOMEM; - - secy->tx_sc.stats = netdev_alloc_pcpu_stats(struct pcpu_tx_sc_stats); - if (!secy->tx_sc.stats) - return -ENOMEM; - - secy->tx_sc.md_dst = metadata_dst_alloc(0, METADATA_MACSEC, GFP_KERNEL); - if (!secy->tx_sc.md_dst) - /* macsec and secy percpu stats will be freed when unregistering - * net_device in macsec_free_netdev() - */ - return -ENOMEM; - if (sci == MACSEC_UNDEF_SCI) sci = dev_to_sci(dev, MACSEC_PORT_ES); @@ -4149,15 +4156,12 @@ static int macsec_add_dev(struct net_device *dev, sci_t sci, u8 icv_len) secy->xpn = DEFAULT_XPN; secy->sci = sci; - secy->tx_sc.md_dst->u.macsec_info.sci = sci; secy->tx_sc.active = true; secy->tx_sc.encoding_sa = DEFAULT_ENCODING_SA; secy->tx_sc.encrypt = DEFAULT_ENCRYPT; secy->tx_sc.send_sci = DEFAULT_SEND_SCI; secy->tx_sc.end_station = false; secy->tx_sc.scb = false; - - return 0; } static struct lock_class_key macsec_netdev_addr_lock_key; @@ -4220,6 +4224,24 @@ static int macsec_newlink(struct net_device *dev, if (rx_handler && rx_handler != macsec_handle_frame) return -EBUSY; + if (is_zero_ether_addr(dev->dev_addr)) + eth_hw_addr_inherit(dev, real_dev); + + if (data && data[IFLA_MACSEC_SCI]) + sci = nla_get_sci(data[IFLA_MACSEC_SCI]); + else if (data && data[IFLA_MACSEC_PORT]) + sci = dev_to_sci(dev, nla_get_be16(data[IFLA_MACSEC_PORT])); + else + sci = dev_to_sci(dev, MACSEC_PORT_ES); + + /* Registration can notify listeners before returning. */ + macsec_init_secy(dev, sci, icv_len); + if (data) { + err = macsec_changelink_common(dev, data); + if (err) + return err; + } + err = register_netdevice(dev); if (err < 0) return err; @@ -4232,31 +4254,11 @@ static int macsec_newlink(struct net_device *dev, if (err < 0) goto unregister; - /* need to be already registered so that ->init has run and - * the MAC addr is set - */ - if (data && data[IFLA_MACSEC_SCI]) - sci = nla_get_sci(data[IFLA_MACSEC_SCI]); - else if (data && data[IFLA_MACSEC_PORT]) - sci = dev_to_sci(dev, nla_get_be16(data[IFLA_MACSEC_PORT])); - else - sci = dev_to_sci(dev, MACSEC_PORT_ES); - if (rx_handler && sci_exists(real_dev, sci)) { err = -EBUSY; goto unlink; } - err = macsec_add_dev(dev, sci, icv_len); - if (err) - goto unlink; - - if (data) { - err = macsec_changelink_common(dev, data); - if (err) - goto del_dev; - } - /* If h/w offloading is available, propagate to the device */ if (macsec_is_offloaded(macsec)) { const struct macsec_ops *ops; diff --git a/drivers/net/mdio/mdio-realtek-rtl9300.c b/drivers/net/mdio/mdio-realtek-rtl9300.c index afd52a1cd7f8..9ce2b7807532 100644 --- a/drivers/net/mdio/mdio-realtek-rtl9300.c +++ b/drivers/net/mdio/mdio-realtek-rtl9300.c @@ -88,6 +88,8 @@ #define RTL9310_SMI_INDRT_ACCESS_BC_PHYID_CTRL 0x0c14 #define RTL9310_BC_PORT_ID GENMASK(10, 5) #define RTL9310_SMI_INDRT_ACCESS_CTRL_1 0x0c04 +#define RTL9310_SMI_INDRT_EXT_PAGE GENMASK(8, 0) +#define RTL9310_SMI_INDRT_EXT_PAGE_NO_CHANGE 0x1ff #define RTL9310_SMI_INDRT_ACCESS_CTRL_2_LOW 0x0c08 #define RTL9310_SMI_INDRT_ACCESS_CTRL_2_HIGH 0x0c0c #define RTL9310_SMI_INDRT_ACCESS_CTRL_3 0x0c10 /* I/O fields flipped */ @@ -325,6 +327,8 @@ static int otto_emdio_9310_read_c22(struct mii_bus *bus, int port, int regnum, u .broadcast = FIELD_PREP(RTL9310_BC_PORT_ID, port), .c22_data = FIELD_PREP(RTL9310_PHY_CTRL_REG_ADDR, regnum) | FIELD_PREP(RTL9310_PHY_CTRL_MAIN_PAGE, RAW_PAGE(priv)), + .ext_page = FIELD_PREP(RTL9310_SMI_INDRT_EXT_PAGE, + RTL9310_SMI_INDRT_EXT_PAGE_NO_CHANGE), }; return otto_emdio_read_cmd(bus, RTL9310_PHY_CTRL_TYPE_C22, &cmd_data, @@ -337,6 +341,8 @@ static int otto_emdio_9310_write_c22(struct mii_bus *bus, int port, int regnum, struct otto_emdio_cmd_regs cmd_data = { .c22_data = FIELD_PREP(RTL9310_PHY_CTRL_REG_ADDR, regnum) | FIELD_PREP(RTL9310_PHY_CTRL_MAIN_PAGE, RAW_PAGE(priv)), + .ext_page = FIELD_PREP(RTL9310_SMI_INDRT_EXT_PAGE, + RTL9310_SMI_INDRT_EXT_PAGE_NO_CHANGE), .io_data = FIELD_PREP(RTL9310_PHY_CTRL_INDATA, value), .port_mask_high = (u32)(BIT_ULL(port) >> 32), .port_mask_low = (u32)(BIT_ULL(port)), diff --git a/drivers/net/ovpn/netlink.c b/drivers/net/ovpn/netlink.c index 4dad85294198..5432bc2eb8e8 100644 --- a/drivers/net/ovpn/netlink.c +++ b/drivers/net/ovpn/netlink.c @@ -100,6 +100,8 @@ static bool ovpn_nl_attr_sockaddr_remote(struct nlattr **attrs, struct sockaddr_in6 *sin6; struct sockaddr_in *sin; struct in6_addr *in6; + struct nlattr *scope; + u32 scope_id = 0; __be16 port = 0; __be32 *in; @@ -114,6 +116,9 @@ static bool ovpn_nl_attr_sockaddr_remote(struct nlattr **attrs, } else if (attrs[OVPN_A_PEER_REMOTE_IPV6]) { ss->ss_family = AF_INET6; in6 = nla_data(attrs[OVPN_A_PEER_REMOTE_IPV6]); + scope = attrs[OVPN_A_PEER_REMOTE_IPV6_SCOPE_ID]; + if (scope) + scope_id = nla_get_u32(scope); } else { return false; } @@ -126,6 +131,7 @@ static bool ovpn_nl_attr_sockaddr_remote(struct nlattr **attrs, if (!ipv6_addr_v4mapped(in6)) { sin6 = (struct sockaddr_in6 *)ss; sin6->sin6_port = port; + sin6->sin6_scope_id = scope_id; memcpy(&sin6->sin6_addr, in6, sizeof(*in6)); break; } @@ -179,6 +185,39 @@ static sa_family_t ovpn_nl_family_get(struct nlattr *addr4, return AF_UNSPEC; } +static int ovpn_nl_peer_check_vpn_addrs(const struct in_addr *addr4, + const struct in6_addr *addr6, + struct genl_info *info) +{ + int addr6_type; + + if (addr4->s_addr == htonl(INADDR_ANY) && ipv6_addr_any(addr6)) { + NL_SET_ERR_MSG_MOD(info->extack, + "at least one VPN IP must be configured in MP mode"); + return -EINVAL; + } + + if (ipv4_is_multicast(addr4->s_addr) || ipv4_is_lbcast(addr4->s_addr) || + ipv4_is_loopback(addr4->s_addr)) { + NL_SET_ERR_MSG_MOD(info->extack, + "VPN IPv4 address must be valid unicast or any"); + return -EADDRNOTAVAIL; + } + + if (!ipv6_addr_any(addr6)) { + addr6_type = ipv6_addr_type(addr6); + + if (!(addr6_type & IPV6_ADDR_UNICAST) || + (addr6_type & (IPV6_ADDR_LOOPBACK | IPV6_ADDR_COMPATv4))) { + NL_SET_ERR_MSG_MOD(info->extack, + "VPN IPv6 address must be valid unicast or any"); + return -EADDRNOTAVAIL; + } + } + + return 0; +} + static int ovpn_nl_peer_precheck(struct ovpn_priv *ovpn, struct genl_info *info, struct nlattr **attrs) @@ -346,8 +385,10 @@ static int ovpn_nl_peer_modify(struct ovpn_peer *peer, struct genl_info *info, int ovpn_nl_peer_new_doit(struct sk_buff *skb, struct genl_info *info) { - struct nlattr *attrs[OVPN_A_PEER_MAX + 1]; + struct in_addr vpn_addr4 = { .s_addr = htonl(INADDR_ANY) }; + struct in6_addr vpn_addr6 = IN6ADDR_ANY_INIT; struct ovpn_priv *ovpn = info->user_ptr[0]; + struct nlattr *attrs[OVPN_A_PEER_MAX + 1]; struct ovpn_socket *ovpn_sock; struct socket *sock = NULL; struct ovpn_peer *peer; @@ -371,11 +412,18 @@ int ovpn_nl_peer_new_doit(struct sk_buff *skb, struct genl_info *info) return -EINVAL; /* in MP mode VPN IPs are required for selecting the right peer */ - if (ovpn->mode == OVPN_MODE_MP && !attrs[OVPN_A_PEER_VPN_IPV4] && - !attrs[OVPN_A_PEER_VPN_IPV6]) { - NL_SET_ERR_MSG_FMT_MOD(info->extack, - "VPN IP must be provided in MP mode"); - return -EINVAL; + if (ovpn->mode == OVPN_MODE_MP) { + if (attrs[OVPN_A_PEER_VPN_IPV4]) + vpn_addr4.s_addr = + nla_get_in_addr(attrs[OVPN_A_PEER_VPN_IPV4]); + if (attrs[OVPN_A_PEER_VPN_IPV6]) + vpn_addr6 = + nla_get_in6_addr(attrs[OVPN_A_PEER_VPN_IPV6]); + + ret = ovpn_nl_peer_check_vpn_addrs(&vpn_addr4, &vpn_addr6, + info); + if (ret < 0) + return ret; } peer_id = nla_get_u32(attrs[OVPN_A_PEER_ID]); @@ -474,8 +522,10 @@ int ovpn_nl_peer_new_doit(struct sk_buff *skb, struct genl_info *info) int ovpn_nl_peer_set_doit(struct sk_buff *skb, struct genl_info *info) { - struct nlattr *attrs[OVPN_A_PEER_MAX + 1]; struct ovpn_priv *ovpn = info->user_ptr[0]; + struct nlattr *attrs[OVPN_A_PEER_MAX + 1]; + struct in6_addr vpn_addr6; + struct in_addr vpn_addr4; struct ovpn_socket *sock; struct ovpn_peer *peer; u32 peer_id; @@ -522,28 +572,58 @@ int ovpn_nl_peer_set_doit(struct sk_buff *skb, struct genl_info *info) rcu_read_unlock(); spin_lock_bh(&ovpn->lock); - ret = ovpn_nl_peer_modify(peer, info, attrs); - if (ret < 0) { - spin_unlock_bh(&ovpn->lock); - ovpn_peer_put(peer); - return ret; + + vpn_addr4 = peer->vpn_addrs.ipv4; + vpn_addr6 = peer->vpn_addrs.ipv6; + + /* reject peer with conflicting VPN address */ + if (attrs[OVPN_A_PEER_VPN_IPV4]) { + vpn_addr4.s_addr = nla_get_in_addr(attrs[OVPN_A_PEER_VPN_IPV4]); + if (ovpn_peer_vpn_addr_conflict4(ovpn, peer, &vpn_addr4)) + goto addr_conflict; } + if (attrs[OVPN_A_PEER_VPN_IPV6]) { + vpn_addr6 = nla_get_in6_addr(attrs[OVPN_A_PEER_VPN_IPV6]); + if (ovpn_peer_vpn_addr_conflict6(ovpn, peer, &vpn_addr6)) + goto addr_conflict; + } + + /* in MP mode VPN IPs are required for selecting the right peer */ + if (ovpn->mode == OVPN_MODE_MP) { + ret = ovpn_nl_peer_check_vpn_addrs(&vpn_addr4, &vpn_addr6, + info); + if (ret < 0) + goto unlock; + } + + ret = ovpn_nl_peer_modify(peer, info, attrs); + if (ret < 0) + goto unlock; /* ret == 1 means that VPN IPv4/6 has been modified and rehashing * is required */ - if (ret > 0) + if (ret > 0) { ovpn_peer_hash_vpn_ip(peer); + ret = 0; + } /* if the remote endpoint was updated, the by_transp_addr hash bucket * also needs to be refreshed, otherwise incoming packets from the new * remote address would fail the lockless lookup */ if (attrs[OVPN_A_PEER_REMOTE_IPV4] || attrs[OVPN_A_PEER_REMOTE_IPV6]) ovpn_peer_hash_transp_addr(peer); + +unlock: spin_unlock_bh(&ovpn->lock); ovpn_peer_put(peer); - return 0; + return ret; +addr_conflict: + NL_SET_ERR_MSG_FMT_MOD(info->extack, + "VPN IP is already assigned to another peer"); + ret = -EADDRINUSE; + goto unlock; } static int ovpn_nl_send_peer(struct sk_buff *skb, const struct genl_info *info, diff --git a/drivers/net/ovpn/peer.c b/drivers/net/ovpn/peer.c index c95656ca7c35..2067825bb5b6 100644 --- a/drivers/net/ovpn/peer.c +++ b/drivers/net/ovpn/peer.c @@ -113,6 +113,7 @@ struct ovpn_peer *ovpn_peer_new(struct ovpn_priv *ovpn, u32 id) RCU_INIT_POINTER(peer->bind, NULL); ovpn_crypto_state_init(&peer->crypto); spin_lock_init(&peer->lock); + seqcount_spinlock_init(&peer->route_key_seq, &peer->lock); kref_init(&peer->refcount); ovpn_peer_stats_init(&peer->vpn_stats); ovpn_peer_stats_init(&peer->link_stats); @@ -199,13 +200,12 @@ static void __ovpn_peer_hash_transp_addr(struct ovpn_peer *peer, */ void ovpn_peer_endpoints_update(struct ovpn_peer *peer, struct sk_buff *skb) { + const void *local_ip = NULL; struct sockaddr_storage ss; struct sockaddr_in6 *sa6; - bool reset_cache = false; struct sockaddr_in *sa; struct ovpn_bind *bind; - const void *local_ip; - size_t salen = 0; + bool floated = false; spin_lock_bh(&peer->lock); bind = rcu_dereference_protected(peer->bind, @@ -232,8 +232,7 @@ void ovpn_peer_endpoints_update(struct ovpn_peer *peer, struct sk_buff *skb) .sin_addr.s_addr = ip_hdr(skb)->saddr, .sin_port = udp_hdr(skb)->source, }; - salen = sizeof(*sa); - reset_cache = true; + floated = true; break; } @@ -245,10 +244,12 @@ void ovpn_peer_endpoints_update(struct ovpn_peer *peer, struct sk_buff *skb) netdev_name(peer->ovpn->dev), peer->id, &bind->local.ipv4.s_addr, &ip_hdr(skb)->daddr); - bind->local.ipv4.s_addr = ip_hdr(skb)->daddr; - reset_cache = true; + local_ip = &ip_hdr(skb)->daddr; + memcpy(&ss, &bind->remote, sizeof(struct sockaddr_in)); + break; } - break; + /* nothing changed */ + goto unlock; case htons(ETH_P_IPV6): /* float check */ if (unlikely(!ovpn_bind_skb_src_match(bind, skb))) { @@ -270,8 +271,7 @@ void ovpn_peer_endpoints_update(struct ovpn_peer *peer, struct sk_buff *skb) ipv6_iface_scope_id(&ipv6_hdr(skb)->saddr, skb->skb_iif), }; - salen = sizeof(*sa6); - reset_cache = true; + floated = true; break; } @@ -284,26 +284,30 @@ void ovpn_peer_endpoints_update(struct ovpn_peer *peer, struct sk_buff *skb) netdev_name(peer->ovpn->dev), peer->id, &bind->local.ipv6, &ipv6_hdr(skb)->daddr); - bind->local.ipv6 = ipv6_hdr(skb)->daddr; - reset_cache = true; + local_ip = &ipv6_hdr(skb)->daddr; + memcpy(&ss, &bind->remote, sizeof(struct sockaddr_in6)); + break; } - break; + /* nothing changed */ + goto unlock; default: goto unlock; } - if (unlikely(reset_cache)) - dst_cache_reset(&peer->dst_cache); - - /* if the peer did not float, we can bail out now */ - if (likely(!salen)) - goto unlock; - if (unlikely(ovpn_peer_reset_sockaddr(peer, (struct sockaddr_storage *)&ss, local_ip) < 0)) goto unlock; + /* reset the cache only after a successful bind update to avoid useless + * cache misses on concurrent TX + */ + dst_cache_reset(&peer->dst_cache); + + /* if only the local address changed, bail out now */ + if (!floated) + goto unlock; + net_dbg_ratelimited("%s: peer %d floated to %pIScp", netdev_name(peer->ovpn->dev), peer->id, &ss); @@ -484,7 +488,7 @@ static struct ovpn_peer *ovpn_peer_get_by_vpn_addr4(struct ovpn_priv *ovpn, * Return: the peer if found or NULL otherwise */ static struct ovpn_peer *ovpn_peer_get_by_vpn_addr6(struct ovpn_priv *ovpn, - struct in6_addr *addr) + const struct in6_addr *addr) { struct hlist_nulls_head *nhead; struct hlist_nulls_node *ntmp; @@ -509,6 +513,64 @@ static struct ovpn_peer *ovpn_peer_get_by_vpn_addr6(struct ovpn_priv *ovpn, return NULL; } +/** + * ovpn_peer_vpn_addr_conflict4 - check if the VPN v4 address is already in use + * @ovpn: the openvpn instance to search + * @peer: peer being added or updated, or NULL + * @addr: VPN IPv4 address to check + * + * Check whether @addr is already assigned to another peer. @peer is ignored + * when found, allowing peer updates that keep an existing address. + * Unspecified addresses are ignored. + * + * Note: the caller must hold @ovpn->lock. + * + * Return: true on conflict, false otherwise. + */ +bool ovpn_peer_vpn_addr_conflict4(struct ovpn_priv *ovpn, + const struct ovpn_peer *peer, + const struct in_addr *addr) +{ + struct ovpn_peer *tmp = NULL; + + lockdep_assert_held(&ovpn->lock); + + /* we don't hash INADDR_ANY, no conflict in that case */ + if (addr->s_addr != htonl(INADDR_ANY)) + tmp = ovpn_peer_get_by_vpn_addr4(ovpn, addr->s_addr); + + return tmp && tmp != peer; +} + +/** + * ovpn_peer_vpn_addr_conflict6 - check if the VPN v6 address is already in use + * @ovpn: the openvpn instance to search + * @peer: peer being added or updated, or NULL + * @addr: VPN IPv6 address to check + * + * Check whether @addr is already assigned to another peer. @peer is ignored + * when found, allowing peer updates that keep an existing address. + * Unspecified addresses are ignored. + * + * Note: the caller must hold @ovpn->lock. + * + * Return: true on conflict, false otherwise. + */ +bool ovpn_peer_vpn_addr_conflict6(struct ovpn_priv *ovpn, + const struct ovpn_peer *peer, + const struct in6_addr *addr) +{ + struct ovpn_peer *tmp = NULL; + + lockdep_assert_held(&ovpn->lock); + + /* we don't hash ::, no conflict in that case */ + if (!ipv6_addr_any(addr)) + tmp = ovpn_peer_get_by_vpn_addr6(ovpn, addr); + + return tmp && tmp != peer; +} + /** * ovpn_peer_transp_match - check if sockaddr and peer binding match * @peer: the peer to get the binding from @@ -990,10 +1052,11 @@ void ovpn_peer_hash_vpn_ip(struct ovpn_peer *peer) if (hlist_unhashed(&peer->hash_entry_id)) return; - if (peer->vpn_addrs.ipv4.s_addr != htonl(INADDR_ANY)) { - /* remove potential old hashing */ - hlist_nulls_del_init_rcu(&peer->hash_entry_addr4); + /* remove potential old hashing */ + hlist_nulls_del_init_rcu(&peer->hash_entry_addr4); + hlist_nulls_del_init_rcu(&peer->hash_entry_addr6); + if (peer->vpn_addrs.ipv4.s_addr != htonl(INADDR_ANY)) { nhead = ovpn_get_hash_head(peer->ovpn->peers->by_vpn_addr4, &peer->vpn_addrs.ipv4, sizeof(peer->vpn_addrs.ipv4)); @@ -1001,9 +1064,6 @@ void ovpn_peer_hash_vpn_ip(struct ovpn_peer *peer) } if (!ipv6_addr_any(&peer->vpn_addrs.ipv6)) { - /* remove potential old hashing */ - hlist_nulls_del_init_rcu(&peer->hash_entry_addr6); - nhead = ovpn_get_hash_head(peer->ovpn->peers->by_vpn_addr6, &peer->vpn_addrs.ipv6, sizeof(peer->vpn_addrs.ipv6)); @@ -1038,6 +1098,13 @@ static int ovpn_peer_add_mp(struct ovpn_priv *ovpn, struct ovpn_peer *peer) goto out; } + /* reject peer with conflicting VPN address */ + if (ovpn_peer_vpn_addr_conflict4(ovpn, NULL, &peer->vpn_addrs.ipv4) || + ovpn_peer_vpn_addr_conflict6(ovpn, NULL, &peer->vpn_addrs.ipv6)) { + ret = -EADDRINUSE; + goto out; + } + bind = rcu_dereference_protected(peer->bind, true); /* peers connected via TCP have bind == NULL */ if (bind) { diff --git a/drivers/net/ovpn/peer.h b/drivers/net/ovpn/peer.h index dfa5c0037e02..1879bfb76992 100644 --- a/drivers/net/ovpn/peer.h +++ b/drivers/net/ovpn/peer.h @@ -10,6 +10,7 @@ #ifndef _NET_OVPN_OVPNPEER_H_ #define _NET_OVPN_OVPNPEER_H_ +#include #include #include @@ -17,6 +18,16 @@ #include "socket.h" #include "stats.h" +/** + * struct ovpn_route_key - route key used for the peer dst cache + * @mark: fwmark used for route lookup + * @sport: UDP source port used for route lookup + */ +struct ovpn_route_key { + u32 mark; + __be16 sport; +}; + /** * struct ovpn_peer - the main remote peer object * @ovpn: main openvpn instance this peer belongs to @@ -45,6 +56,8 @@ * @tcp.sk_cb.ops: pointer to the original prot_ops object (TCP only) * @crypto: the crypto configuration (ciphers, keys, etc..) * @dst_cache: cache for dst_entry used to send to peer + * @route_key: route key matching the current dst cache contents + * @route_key_seq: seqcount protecting lockless route_key reads * @bind: remote peer binding * @keepalive_interval: seconds after which a new keepalive should be sent * @keepalive_xmit_exp: future timestamp when next keepalive should be sent @@ -55,7 +68,7 @@ * @vpn_stats: per-peer in-VPN TX/RX stats * @link_stats: per-peer link/transport TX/RX stats * @delete_reason: why peer was deleted (i.e. timeout, transport error, ..) - * @lock: protects binding to peer (bind) and keepalive* fields + * @lock: protects binding to peer (bind), route_key and keepalive* fields * @refcount: reference counter * @rcu: used to free peer in an RCU safe way * @release_entry: entry for the socket release list @@ -99,6 +112,8 @@ struct ovpn_peer { } tcp; struct ovpn_crypto_state crypto; struct dst_cache dst_cache; + struct ovpn_route_key route_key; + seqcount_spinlock_t route_key_seq; struct ovpn_bind __rcu *bind; unsigned long keepalive_interval; unsigned long keepalive_xmit_exp; @@ -109,7 +124,7 @@ struct ovpn_peer { struct ovpn_peer_stats vpn_stats; struct ovpn_peer_stats link_stats; enum ovpn_del_peer_reason delete_reason; - spinlock_t lock; /* protects bind and keepalive* */ + spinlock_t lock; /* protects bind, route_key and keepalive* */ struct kref refcount; struct rcu_head rcu; struct llist_node release_entry; @@ -149,6 +164,12 @@ struct ovpn_peer *ovpn_peer_get_by_transp_addr(struct ovpn_priv *ovpn, struct ovpn_peer *ovpn_peer_get_by_id(struct ovpn_priv *ovpn, u32 peer_id); struct ovpn_peer *ovpn_peer_get_by_dst(struct ovpn_priv *ovpn, struct sk_buff *skb); +bool ovpn_peer_vpn_addr_conflict4(struct ovpn_priv *ovpn, + const struct ovpn_peer *peer, + const struct in_addr *addr); +bool ovpn_peer_vpn_addr_conflict6(struct ovpn_priv *ovpn, + const struct ovpn_peer *peer, + const struct in6_addr *addr); void ovpn_peer_hash_vpn_ip(struct ovpn_peer *peer); void ovpn_peer_hash_transp_addr(struct ovpn_peer *peer); bool ovpn_peer_check_by_src(struct ovpn_priv *ovpn, struct sk_buff *skb, diff --git a/drivers/net/ovpn/udp.c b/drivers/net/ovpn/udp.c index 7f69e8890b5b..055cdb1bee13 100644 --- a/drivers/net/ovpn/udp.c +++ b/drivers/net/ovpn/udp.c @@ -131,6 +131,77 @@ static int ovpn_udp_encap_recv(struct sock *sk, struct sk_buff *skb) return 0; } +static bool ovpn_route_key_equal(const struct ovpn_route_key *a, + const struct ovpn_route_key *b) +{ + return a->mark == b->mark && a->sport == b->sport; +} + +/** + * ovpn_dst_cache_check_key - reset peer dst cache after key changes + * @peer: the peer owning the dst cache + * @cache: the cache that might need to be reset + * @key: the route key for the packet being transmitted + * + * Reset the peer dst cache if it was populated for a different route key. + */ +static void ovpn_dst_cache_check_key(struct ovpn_peer *peer, + struct dst_cache *cache, + const struct ovpn_route_key *key) +{ + struct ovpn_route_key old_key; + unsigned int seq; + + /* snapshot the saved key before deciding whether the cache matches */ + do { + seq = read_seqcount_begin(&peer->route_key_seq); + old_key = peer->route_key; + } while (read_seqcount_retry(&peer->route_key_seq, seq)); + + /* nothing changed: the current cache can be reused */ + if (likely(ovpn_route_key_equal(&old_key, key))) + return; + + /* recheck under lock because another path may have updated the key */ + spin_lock_bh(&peer->lock); + if (!ovpn_route_key_equal(&peer->route_key, key)) { + write_seqcount_begin(&peer->route_key_seq); + peer->route_key = *key; + dst_cache_reset(cache); + write_seqcount_end(&peer->route_key_seq); + } + spin_unlock_bh(&peer->lock); +} + +/** + * ovpn_dst_cache_current - check whether a route lookup matches peer state + * @peer: the peer owning the bind and dst cache + * @bind: the RCU bind used for the route lookup + * @key: the route key used for the route lookup + * + * Check that @bind is still the current peer bind and that @key still matches + * the peer route key. The caller must hold @peer->lock. The TX path keeps + * @bind inside an RCU read-side critical section, so pointer identity is enough + * to detect whether the bind was replaced while the route lookup was running. + * + * Return: true if the lookup result still matches the current peer state and + * may update the dst cache or replace the bind. + */ +static bool ovpn_dst_cache_current(const struct ovpn_peer *peer, + const struct ovpn_bind *bind, + const struct ovpn_route_key *key) +{ + const struct ovpn_bind *curr_bind; + + lockdep_assert_held(&peer->lock); + + curr_bind = rcu_dereference_protected(peer->bind, + lockdep_is_held(&peer->lock)); + + return curr_bind == bind && + ovpn_route_key_equal(key, &peer->route_key); +} + /** * ovpn_udp4_output - send IPv4 packet over udp socket * @peer: the destination peer @@ -138,21 +209,26 @@ static int ovpn_udp_encap_recv(struct sock *sk, struct sk_buff *skb) * @cache: dst cache * @sk: the socket to send the packet over * @skb: the packet to send + * @key: the route key snapshot used for cache validation and flow lookup * * Return: 0 on success or a negative error code otherwise */ static int ovpn_udp4_output(struct ovpn_peer *peer, struct ovpn_bind *bind, struct dst_cache *cache, struct sock *sk, - struct sk_buff *skb) + struct sk_buff *skb, + const struct ovpn_route_key *key) { + struct sockaddr_storage remote; + struct in_addr local = {}; + bool reset_local = false; struct rtable *rt; struct flowi4 fl = { .saddr = bind->local.ipv4.s_addr, .daddr = bind->remote.in4.sin_addr.s_addr, - .fl4_sport = inet_sk(sk)->inet_sport, + .fl4_sport = key->sport, .fl4_dport = bind->remote.in4.sin_port, .flowi4_proto = sk->sk_protocol, - .flowi4_mark = sk->sk_mark, + .flowi4_mark = key->mark, }; int ret; @@ -161,26 +237,19 @@ static int ovpn_udp4_output(struct ovpn_peer *peer, struct ovpn_bind *bind, if (rt) goto transmit; - if (unlikely(!inet_confirm_addr(sock_net(sk), NULL, 0, fl.saddr, - RT_SCOPE_HOST))) { - /* we may end up here when the cached address is not usable - * anymore. In this case we reset address/cache and perform a - * new look up + if (fl.saddr && unlikely(!inet_confirm_addr(sock_net(sk), NULL, 0, + fl.saddr, RT_SCOPE_HOST))) { + /* The learned local address is not usable anymore. + * Retry with source address autoselection. */ fl.saddr = 0; - spin_lock_bh(&peer->lock); - bind->local.ipv4.s_addr = 0; - spin_unlock_bh(&peer->lock); - dst_cache_reset(cache); + reset_local = true; } rt = ip_route_output_flow(sock_net(sk), &fl, sk); if (IS_ERR(rt) && PTR_ERR(rt) == -EINVAL) { fl.saddr = 0; - spin_lock_bh(&peer->lock); - bind->local.ipv4.s_addr = 0; - spin_unlock_bh(&peer->lock); - dst_cache_reset(cache); + reset_local = true; rt = ip_route_output_flow(sock_net(sk), &fl, sk); } @@ -193,7 +262,30 @@ static int ovpn_udp4_output(struct ovpn_peer *peer, struct ovpn_bind *bind, ret); goto err; } - dst_cache_set_ip4(cache, &rt->dst, fl.saddr); + + /* avoid storing a stale cache or local address */ + spin_lock_bh(&peer->lock); + if (likely(ovpn_dst_cache_current(peer, bind, key))) { + if (!reset_local) { + dst_cache_set_ip4(cache, &rt->dst, fl.saddr); + spin_unlock_bh(&peer->lock); + goto transmit; + } + + /* invalidate per-CPU dst entries that may still carry + * the stale source + */ + dst_cache_reset(cache); + + /* preserve the current remote */ + memcpy(&remote, &bind->remote, sizeof(struct sockaddr_in)); + /* The current packet already has a valid wildcard-source route. + * If replacing the bind fails, leave the stale local in place; + * a later cache miss will retry the repair. + */ + ovpn_peer_reset_sockaddr(peer, &remote, &local); + } + spin_unlock_bh(&peer->lock); transmit: udp_tunnel_xmit_skb(rt, sk, skb, fl.saddr, fl.daddr, 0, @@ -213,23 +305,28 @@ static int ovpn_udp4_output(struct ovpn_peer *peer, struct ovpn_bind *bind, * @cache: dst cache * @sk: the socket to send the packet over * @skb: the packet to send + * @key: the route key snapshot used for cache validation and flow lookup * * Return: 0 on success or a negative error code otherwise */ static int ovpn_udp6_output(struct ovpn_peer *peer, struct ovpn_bind *bind, struct dst_cache *cache, struct sock *sk, - struct sk_buff *skb) + struct sk_buff *skb, + const struct ovpn_route_key *key) { + struct in6_addr local = in6addr_any; + struct sockaddr_storage remote; + bool reset_local = false; struct dst_entry *dst; int ret; struct flowi6 fl = { .saddr = bind->local.ipv6, .daddr = bind->remote.in6.sin6_addr, - .fl6_sport = inet_sk(sk)->inet_sport, + .fl6_sport = key->sport, .fl6_dport = bind->remote.in6.sin6_port, .flowi6_proto = sk->sk_protocol, - .flowi6_mark = sk->sk_mark, + .flowi6_mark = key->mark, .flowi6_oif = bind->remote.in6.sin6_scope_id, }; @@ -238,16 +335,13 @@ static int ovpn_udp6_output(struct ovpn_peer *peer, struct ovpn_bind *bind, if (dst) goto transmit; - if (unlikely(!ipv6_chk_addr(sock_net(sk), &fl.saddr, NULL, 0))) { - /* we may end up here when the cached address is not usable - * anymore. In this case we reset address/cache and perform a - * new look up + if (!ipv6_addr_any(&fl.saddr) && + unlikely(!ipv6_chk_addr(sock_net(sk), &fl.saddr, NULL, 0))) { + /* The learned local address is not usable anymore. + * Retry with source address autoselection. */ fl.saddr = in6addr_any; - spin_lock_bh(&peer->lock); - bind->local.ipv6 = in6addr_any; - spin_unlock_bh(&peer->lock); - dst_cache_reset(cache); + reset_local = true; } dst = ip6_dst_lookup_flow(sock_net(sk), sk, &fl, NULL); @@ -258,7 +352,30 @@ static int ovpn_udp6_output(struct ovpn_peer *peer, struct ovpn_bind *bind, &bind->remote.in6, ret); goto err; } - dst_cache_set_ip6(cache, dst, &fl.saddr); + + /* avoid storing a stale cache or local address */ + spin_lock_bh(&peer->lock); + if (likely(ovpn_dst_cache_current(peer, bind, key))) { + if (!reset_local) { + dst_cache_set_ip6(cache, dst, &fl.saddr); + spin_unlock_bh(&peer->lock); + goto transmit; + } + + /* invalidate per-CPU dst entries that may still carry + * the stale source + */ + dst_cache_reset(cache); + + /* preserve the current remote */ + memcpy(&remote, &bind->remote, sizeof(struct sockaddr_in6)); + /* The current packet already has a valid wildcard-source route. + * If replacing the bind fails, leave the stale local in place; + * a later cache miss will retry the repair. + */ + ovpn_peer_reset_sockaddr(peer, &remote, &local); + } + spin_unlock_bh(&peer->lock); transmit: /* user IPv6 packets may be larger than the transport interface @@ -287,6 +404,7 @@ static int ovpn_udp6_output(struct ovpn_peer *peer, struct ovpn_bind *bind, * @cache: dst cache * @sk: the socket to send the packet over * @skb: the packet to send + * @key: route key snapshot used for cache validation and flow lookup * * rcu_read_lock should be held on entry. * On return, the skb is consumed. @@ -294,7 +412,8 @@ static int ovpn_udp6_output(struct ovpn_peer *peer, struct ovpn_bind *bind, * Return: 0 on success or a negative error code otherwise */ static int ovpn_udp_output(struct ovpn_peer *peer, struct dst_cache *cache, - struct sock *sk, struct sk_buff *skb) + struct sock *sk, struct sk_buff *skb, + struct ovpn_route_key *key) { struct ovpn_bind *bind; int ret; @@ -314,11 +433,11 @@ static int ovpn_udp_output(struct ovpn_peer *peer, struct dst_cache *cache, switch (bind->remote.in4.sin_family) { case AF_INET: - ret = ovpn_udp4_output(peer, bind, cache, sk, skb); + ret = ovpn_udp4_output(peer, bind, cache, sk, skb, key); break; #if IS_ENABLED(CONFIG_IPV6) case AF_INET6: - ret = ovpn_udp6_output(peer, bind, cache, sk, skb); + ret = ovpn_udp6_output(peer, bind, cache, sk, skb, key); break; #endif default: @@ -340,15 +459,21 @@ static int ovpn_udp_output(struct ovpn_peer *peer, struct dst_cache *cache, void ovpn_udp_send_skb(struct ovpn_peer *peer, struct sock *sk, struct sk_buff *skb) { + struct ovpn_route_key key = { + .mark = READ_ONCE(sk->sk_mark), + .sport = READ_ONCE(inet_sk(sk)->inet_sport), + }; int ret; skb->dev = peer->ovpn->dev; - skb->mark = READ_ONCE(sk->sk_mark); + skb->mark = key.mark; /* no checksum performed at this layer */ skb->ip_summed = CHECKSUM_NONE; + ovpn_dst_cache_check_key(peer, &peer->dst_cache, &key); + /* crypto layer -> transport (UDP) */ - ret = ovpn_udp_output(peer, &peer->dst_cache, sk, skb); + ret = ovpn_udp_output(peer, &peer->dst_cache, sk, skb, &key); if (unlikely(ret < 0)) kfree_skb(skb); } diff --git a/drivers/net/pcs/pcs-rzn1-miic.c b/drivers/net/pcs/pcs-rzn1-miic.c index 2b72fa98ddf1..3aa513467060 100644 --- a/drivers/net/pcs/pcs-rzn1-miic.c +++ b/drivers/net/pcs/pcs-rzn1-miic.c @@ -270,6 +270,9 @@ static struct miic_port *phylink_pcs_to_miic_port(struct phylink_pcs *pcs) static void miic_unlock_regs(struct miic *miic) { + /* Clear protection state */ + writel(0x0000, miic->base + MIIC_PRCMD); + /* Unprotect register writes */ writel(0x00A5, miic->base + MIIC_PRCMD); writel(0x0001, miic->base + MIIC_PRCMD); @@ -683,7 +686,8 @@ static int miic_parse_dt(struct miic *miic, u32 *mode_cfg) if (!dt_val) return -ENOMEM; - memset(dt_val, MIIC_MODCTRL_CONF_NONE, sizeof(*dt_val)); + memset(dt_val, MIIC_MODCTRL_CONF_NONE, + sizeof(*dt_val) * miic->of_data->conf_conv_count); if (of_property_read_u32(np, "renesas,miic-switch-portin", &conf) == 0) dt_val[0] = conf; diff --git a/drivers/net/pcs/pcs-xpcs.c b/drivers/net/pcs/pcs-xpcs.c index 0337e2bcc012..b415b93d77c1 100644 --- a/drivers/net/pcs/pcs-xpcs.c +++ b/drivers/net/pcs/pcs-xpcs.c @@ -1545,8 +1545,10 @@ static int xpcs_init_clks(struct dw_xpcs *xpcs) return dev_err_probe(dev, ret, "Failed to get clocks\n"); ret = clk_bulk_prepare_enable(DW_XPCS_NUM_CLKS, xpcs->clks); - if (ret) + if (ret) { + clk_bulk_put(DW_XPCS_NUM_CLKS, xpcs->clks); return dev_err_probe(dev, ret, "Failed to enable clocks\n"); + } return 0; } diff --git a/drivers/net/pfcp.c b/drivers/net/pfcp.c index e1cca779d2ec..3ecd43cafdf0 100644 --- a/drivers/net/pfcp.c +++ b/drivers/net/pfcp.c @@ -105,6 +105,9 @@ static int pfcp_encap_recv(struct sock *sk, struct sk_buff *skb) static void pfcp_del_sock(struct pfcp_dev *pfcp) { + if (!pfcp->sk) + return; + udp_tunnel_sock_release(pfcp->sk); pfcp->sk = NULL; } diff --git a/drivers/net/phy/aquantia/aquantia_main.c b/drivers/net/phy/aquantia/aquantia_main.c index e95d391a6dc2..096407bebd52 100644 --- a/drivers/net/phy/aquantia/aquantia_main.c +++ b/drivers/net/phy/aquantia/aquantia_main.c @@ -544,7 +544,7 @@ static int aqr_gen1_read_status(struct phy_device *phydev) if (ret) return ret; - if (!phydev->link || phydev->autoneg == AUTONEG_DISABLE) + if (!phydev->link) return 0; /* The status register is not immediately correct on line side link up. @@ -594,6 +594,9 @@ static int aqr_gen1_read_status(struct phy_device *phydev) phydev->interface = aqr_translate_interface(phydev, interface); + if (phydev->autoneg == AUTONEG_DISABLE) + return 0; + /* Read rate from vendor register */ return aqr_gen1_read_rate(phydev); } diff --git a/drivers/net/phy/intel-xway.c b/drivers/net/phy/intel-xway.c index afbcec711744..3cee31bb931f 100644 --- a/drivers/net/phy/intel-xway.c +++ b/drivers/net/phy/intel-xway.c @@ -16,6 +16,11 @@ #define XWAY_MDIO_ISTAT 0x1A /* interrupt status */ #define XWAY_MDIO_LED 0x1B /* led control */ +#define XWAY_MDIO_GCTRL_TM_MASK GENMASK(15, 13) +#define XWAY_MDIO_GCTRL_TM(mode) FIELD_PREP(XWAY_MDIO_GCTRL_TM_MASK, (mode)) +#define XWAY_MDIO_GCTRL_TM_NOP XWAY_MDIO_GCTRL_TM(0) /* Normal operation */ +#define XWAY_MDIO_GCTRL_TM_CDIAG XWAY_MDIO_GCTRL_TM(6) /* Cable diagnostics */ + #define XWAY_MDIO_ERRCNT_SEL GENMASK(11, 8) #define XWAY_MDIO_ERRCNT_COUNT GENMASK(7, 0) #define XWAY_MDIO_ERRCNT_SEL_RXERR 0 @@ -326,6 +331,28 @@ static int xway_gphy_probe(struct phy_device *phydev) return 0; } +static int xway_11g_int_config_init(struct phy_device *phydev) +{ + int err; + + /* An issue has been sporadically observed after device power-on on the + * first link-up attempt in 100BASE-TX mode resulting in either the + * link-up taking a long time, or failing to link-up altogether. + * + * Workaround: + * After power-on, enable Cable Diagnostic Mode for all ports and + * disable it. + */ + err = phy_modify(phydev, MII_CTRL1000, XWAY_MDIO_GCTRL_TM_MASK, XWAY_MDIO_GCTRL_TM_CDIAG); + if (err) + return err; + err = phy_modify(phydev, MII_CTRL1000, XWAY_MDIO_GCTRL_TM_MASK, XWAY_MDIO_GCTRL_TM_NOP); + if (err) + return err; + + return xway_gphy_config_init(phydev); +} + static int xway_gphy14_config_aneg(struct phy_device *phydev) { int reg, err; @@ -735,7 +762,7 @@ static struct phy_driver xway_gphy[] = { .phy_id_mask = 0xffffffff, .name = "Intel XWAY PHY11G (xRX v1.2 integrated)", /* PHY_GBIT_FEATURES */ - .config_init = xway_gphy_config_init, + .config_init = xway_11g_int_config_init, .probe = xway_gphy_probe, .handle_interrupt = xway_gphy_handle_interrupt, .config_intr = xway_gphy_config_intr, diff --git a/drivers/net/phy/micrel.c b/drivers/net/phy/micrel.c index ae830781824b..5c8461db7b4b 100644 --- a/drivers/net/phy/micrel.c +++ b/drivers/net/phy/micrel.c @@ -6344,6 +6344,7 @@ static int lanphy_write_reg_data(struct phy_device *phydev, data->val); if (ret) break; + data++; } return ret; diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c index a1458da8111b..1bbcf46c8356 100644 --- a/drivers/net/phy/phylink.c +++ b/drivers/net/phy/phylink.c @@ -2129,7 +2129,6 @@ static int phylink_bringup_phy(struct phylink *pl, struct phy_device *phy, mutex_lock(&pl->phydev_mutex); mutex_lock(&phy->lock); mutex_lock(&pl->state_mutex); - pl->phydev = phy; pl->phy_state.interface = interface; pl->phy_state.pause = MLO_PAUSE_NONE; pl->phy_state.speed = SPEED_UNKNOWN; @@ -2196,10 +2195,25 @@ static int phylink_bringup_phy(struct phylink *pl, struct phy_device *phy, ret = 0; } - if (ret == 0 && phy_interrupt_is_valid(phy)) + if (ret) + return ret; + + /* Nothing below can fail, so the PHY can be recorded now. Doing it + * here rather than above keeps a failed bringup from leaving + * pl->phydev pointing at a PHY the caller is about to detach. + */ + mutex_lock(&pl->phydev_mutex); + mutex_lock(&phy->lock); + mutex_lock(&pl->state_mutex); + pl->phydev = phy; + mutex_unlock(&pl->state_mutex); + mutex_unlock(&phy->lock); + mutex_unlock(&pl->phydev_mutex); + + if (phy_interrupt_is_valid(phy)) phy_request_interrupt(phy); - return ret; + return 0; } static int phylink_attach_phy(struct phylink *pl, struct phy_device *phy, diff --git a/drivers/net/phy/qcom/at803x.c b/drivers/net/phy/qcom/at803x.c index 6872dbf77856..cacbadf1f484 100644 --- a/drivers/net/phy/qcom/at803x.c +++ b/drivers/net/phy/qcom/at803x.c @@ -1051,11 +1051,15 @@ static int ipq5018_config_init(struct phy_device *phydev) if (priv->set_short_cable_dac) { /* setting MDAC (Multi-level Digital-to-Analog Converter) in MMD1 */ phy_modify_mmd(phydev, MDIO_MMD_PMAPMD, IPQ5018_PHY_MMD1_MDAC, - IPQ5018_PHY_DAC_MASK, IPQ5018_PHY_MMD1_MDAC_VAL); + IPQ5018_PHY_DAC_MASK, + FIELD_PREP(IPQ5018_PHY_DAC_MASK, + IPQ5018_PHY_MMD1_MDAC_VAL)); /* setting EDAC (Error-detection and Correction) in debug register */ at803x_debug_reg_mask(phydev, IPQ5018_PHY_DEBUG_EDAC, - IPQ5018_PHY_DAC_MASK, IPQ5018_PHY_DEBUG_EDAC_VAL); + IPQ5018_PHY_DAC_MASK, + FIELD_PREP(IPQ5018_PHY_DAC_MASK, + IPQ5018_PHY_DEBUG_EDAC_VAL)); } return 0; diff --git a/drivers/net/usb/catc.c b/drivers/net/usb/catc.c index 96e82f94edcf..39b678f175dd 100644 --- a/drivers/net/usb/catc.c +++ b/drivers/net/usb/catc.c @@ -233,17 +233,26 @@ static void catc_rx_done(struct urb *urb) } do { - if(!catc->is_f5u011) { - pkt_len = le16_to_cpup((__le16*)pkt_start); - if (pkt_len > urb->actual_length) { + int remaining = urb->actual_length - + (pkt_start - (u8 *)urb->transfer_buffer); + + if (!catc->is_f5u011) { + if (remaining < pkt_offset) { catc->netdev->stats.rx_length_errors++; catc->netdev->stats.rx_errors++; break; } + pkt_len = le16_to_cpup((__le16 *)pkt_start); } else { pkt_len = urb->actual_length; } + if (pkt_len < ETH_HLEN || pkt_len + pkt_offset > remaining) { + catc->netdev->stats.rx_length_errors++; + catc->netdev->stats.rx_errors++; + break; + } + if (!(skb = dev_alloc_skb(pkt_len))) return; diff --git a/drivers/net/usb/cdc_mbim.c b/drivers/net/usb/cdc_mbim.c index 877fb0ed7d3d..a7010a0664c7 100644 --- a/drivers/net/usb/cdc_mbim.c +++ b/drivers/net/usb/cdc_mbim.c @@ -635,6 +635,11 @@ static const struct usb_device_id mbim_devs[] = { .driver_info = (unsigned long)&cdc_mbim_info, }, + /* MeiG Smart SRM821 ZLP conformance */ + { USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d53, USB_CLASS_COMM, USB_CDC_SUBCLASS_MBIM, USB_CDC_PROTO_NONE), + .driver_info = (unsigned long)&cdc_mbim_info, + }, + /* Some Huawei devices, ME906s-158 (12d1:15c1) and E3372 * (12d1:157d), are known to fail unless the NDP is placed * after the IP packets. Applying the quirk to all Huawei diff --git a/drivers/net/usb/lan78xx.c b/drivers/net/usb/lan78xx.c index cb782d81d84f..5655941f1478 100644 --- a/drivers/net/usb/lan78xx.c +++ b/drivers/net/usb/lan78xx.c @@ -5239,10 +5239,12 @@ static bool lan78xx_submit_deferred_urbs(struct lan78xx_net *dev) !netif_carrier_ok(dev->net) || pipe_halted) { lan78xx_release_tx_buf(dev, skb); + usb_put_urb(urb); continue; } ret = usb_submit_urb(urb, GFP_ATOMIC); + usb_put_urb(urb); if (ret == 0) { netif_trans_update(dev->net); diff --git a/drivers/net/usb/qmi_wwan.c b/drivers/net/usb/qmi_wwan.c index f51cf9cb9421..a44d1b9756a1 100644 --- a/drivers/net/usb/qmi_wwan.c +++ b/drivers/net/usb/qmi_wwan.c @@ -1086,6 +1086,7 @@ static const struct usb_device_id products[] = { {QMI_MATCH_FF_FF_FF(0x2c7c, 0x0125)}, /* Quectel EC25, EC20 R2.0 Mini PCIe */ {QMI_MATCH_FF_FF_FF(0x2c7c, 0x013d)}, /* Quectel RG660QB */ {QMI_MATCH_FF_FF_FF(0x2c7c, 0x0306)}, /* Quectel EP06/EG06/EM06 */ + {QMI_MATCH_FF_FF_FF(0x2c7c, 0x030b)}, /* Quectel EM060K/EG120K-EA */ {QMI_MATCH_FF_FF_FF(0x2c7c, 0x0512)}, /* Quectel EG12/EM12 */ {QMI_MATCH_FF_FF_FF(0x2c7c, 0x0620)}, /* Quectel EM160R-GL */ {QMI_MATCH_FF_FF_FF(0x2c7c, 0x0800)}, /* Quectel RM500Q-GL */ @@ -1454,6 +1455,7 @@ static const struct usb_device_id products[] = { {QMI_QUIRK_SET_DTR(0x1546, 0x1312, 4)}, /* u-blox LARA-R6 01B */ {QMI_QUIRK_SET_DTR(0x1546, 0x1342, 4)}, /* u-blox LARA-L6 */ {QMI_QUIRK_SET_DTR(0x33f8, 0x0104, 4)}, /* Rolling RW101 RMNET */ + {QMI_QUIRK_SET_DTR(0x33f8, 0x9300, 8)}, /* Rolling RN947R RMNET */ {QMI_FIXED_INTF(0x2dee, 0x4d22, 5)}, /* MeiG Smart SRM825L */ {QMI_QUIRK_SET_DTR(0x2dee, 0x4d63, 0)}, /* MeiG SRM813Q w/ Modem(PPP) */ {QMI_QUIRK_SET_DTR(0x2dee, 0x4d64, 0)}, /* MeiG SRM813Q w/ NMEA */ diff --git a/drivers/net/usb/sr9700.c b/drivers/net/usb/sr9700.c index 937e6fef3ac6..50981a28376a 100644 --- a/drivers/net/usb/sr9700.c +++ b/drivers/net/usb/sr9700.c @@ -355,7 +355,8 @@ static int sr9700_rx_fixup(struct usbnet *dev, struct sk_buff *skb) /* ignore the CRC length */ len = (skb->data[1] | (skb->data[2] << 8)) - 4; - if (len > ETH_FRAME_LEN || len > skb->len || len < 0) + if (len > ETH_FRAME_LEN || len < 0 || + len > skb->len - SR_RX_OVERHEAD) return 0; /* the last packet of current skb */ diff --git a/drivers/net/veth.c b/drivers/net/veth.c index 6ed3ee81153f..71227d0389aa 100644 --- a/drivers/net/veth.c +++ b/drivers/net/veth.c @@ -1054,6 +1054,7 @@ static int __veth_napi_enable_range(struct net_device *dev, int start, int end) for (i = start; i < end; i++) { struct veth_rq *rq = &priv->rq[i]; + rcu_assign_pointer(rq->xdp_prog, priv->_xdp_prog); napi_enable(&rq->xdp_napi); rcu_assign_pointer(priv->rq[i].napi, &priv->rq[i].xdp_napi); } @@ -1088,6 +1089,7 @@ static void veth_napi_del_range(struct net_device *dev, int start, int end) rcu_assign_pointer(priv->rq[i].napi, NULL); napi_disable(&rq->xdp_napi); + rcu_assign_pointer(rq->xdp_prog, NULL); __netif_napi_del(&rq->xdp_napi); } synchronize_net(); diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index e34c52d059d3..bf82ef9874ab 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -3349,6 +3349,14 @@ static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev) else virtqueue_disable_cb(sq->vq); + if (!use_napi && + unlikely(skb_orphan_frags(skb, GFP_ATOMIC))) { + DEV_STATS_INC(dev, tx_dropped); + dev_kfree_skb_any(skb); + kick = !xmit_more || netif_xmit_stopped(txq); + goto kick_vq; + } + /* timestamp packet in software */ skb_tx_timestamp(skb); @@ -3381,6 +3389,7 @@ static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev) kick = use_napi ? __netdev_tx_sent_queue(txq, skb->len, xmit_more) : !xmit_more || netif_xmit_stopped(txq); +kick_vq: if (kick) { if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) { u64_stats_update_begin(&sq->stats.syncp); diff --git a/drivers/net/vrf.c b/drivers/net/vrf.c index a0557a3a7026..d4dc6d690a75 100644 --- a/drivers/net/vrf.c +++ b/drivers/net/vrf.c @@ -1175,8 +1175,6 @@ static int vrf_prepare_mac_header(struct sk_buff *skb, skb->protocol = eth->h_proto; skb->pkt_type = PACKET_HOST; - skb_postpush_rcsum(skb, skb->data, ETH_HLEN); - skb_pull_inline(skb, ETH_HLEN); return 0; diff --git a/drivers/net/vxlan/vxlan_core.c b/drivers/net/vxlan/vxlan_core.c index c1d54339fa2b..3390e341d1e5 100644 --- a/drivers/net/vxlan/vxlan_core.c +++ b/drivers/net/vxlan/vxlan_core.c @@ -1958,13 +1958,15 @@ static struct sk_buff *vxlan_na_create(struct sk_buff *request, struct ipv6hdr *pip6; u8 *daddr; int na_olen = 8; /* opt hdr + ETH_ALEN for target */ + int headroom; int ns_olen; int i, len; if (dev == NULL || !pskb_may_pull(request, request->len)) return NULL; - len = LL_RESERVED_SPACE(dev) + sizeof(struct ipv6hdr) + + headroom = LL_RESERVED_SPACE(dev); + len = headroom + sizeof(struct ipv6hdr) + sizeof(*na) + na_olen + dev->needed_tailroom; reply = alloc_skb(len, GFP_ATOMIC); if (reply == NULL) @@ -1972,7 +1974,7 @@ static struct sk_buff *vxlan_na_create(struct sk_buff *request, reply->protocol = htons(ETH_P_IPV6); reply->dev = dev; - skb_reserve(reply, LL_RESERVED_SPACE(request->dev)); + skb_reserve(reply, headroom); skb_push(reply, sizeof(struct ethhdr)); skb_reset_mac_header(reply); diff --git a/drivers/net/wireless/ath/ath11k/mac.c b/drivers/net/wireless/ath/ath11k/mac.c index ae91b57c8422..c1fa42edd363 100644 --- a/drivers/net/wireless/ath/ath11k/mac.c +++ b/drivers/net/wireless/ath/ath11k/mac.c @@ -6716,6 +6716,7 @@ static int ath11k_mac_op_start(struct ieee80211_hw *hw) ar->num_started_vdevs = 0; ar->num_created_vdevs = 0; ar->num_peers = 0; + ar->num_stations = 0; ar->allocated_vdev_map = 0; /* Configure monitor status ring with default rx_filter to get rx status diff --git a/drivers/net/wireless/ath/ath9k/hif_usb.c b/drivers/net/wireless/ath/ath9k/hif_usb.c index d3491ff08e6e..7d2f292e20c5 100644 --- a/drivers/net/wireless/ath/ath9k/hif_usb.c +++ b/drivers/net/wireless/ath/ath9k/hif_usb.c @@ -14,6 +14,7 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include #include #include "htc.h" @@ -328,11 +329,14 @@ static int __hif_usb_tx(struct hif_device_usb *hif_dev) tx_skb_cnt = min_t(u16, hif_dev->tx.tx_skb_cnt, MAX_TX_AGGR_NUM); for (i = 0; i < tx_skb_cnt; i++) { + nskb = skb_peek(&hif_dev->tx.tx_skb_queue); + if (!nskb) + break; + + if (tx_buf->offset + nskb->len + 4 > MAX_TX_BUF_SIZE) + break; + nskb = __skb_dequeue(&hif_dev->tx.tx_skb_queue); - - /* Should never be NULL */ - BUG_ON(!nskb); - hif_dev->tx.tx_skb_cnt--; buf = tx_buf->buf; @@ -342,13 +346,8 @@ static int __hif_usb_tx(struct hif_device_usb *hif_dev) *hdr++ = cpu_to_le16(ATH_USB_TX_STREAM_MODE_TAG); buf += 4; memcpy(buf, nskb->data, nskb->len); - tx_buf->len = nskb->len + 4; - - if (i < (tx_skb_cnt - 1)) - tx_buf->offset += (((tx_buf->len - 1) / 4) + 1) * 4; - - if (i == (tx_skb_cnt - 1)) - tx_buf->len += tx_buf->offset; + tx_buf->len = tx_buf->offset + nskb->len + 4; + tx_buf->offset += round_up(nskb->len + 4, 4); __skb_queue_tail(&tx_buf->skb_queue, nskb); TX_STAT_INC(hif_dev, skb_queued); diff --git a/drivers/net/wireless/ath/ath9k/htc.h b/drivers/net/wireless/ath/ath9k/htc.h index 6c33e898b300..4d2b75d0212e 100644 --- a/drivers/net/wireless/ath/ath9k/htc.h +++ b/drivers/net/wireless/ath/ath9k/htc.h @@ -281,7 +281,6 @@ struct ath9k_htc_rxbuf { struct ath9k_htc_rx { struct list_head rxbuf; spinlock_t rxbuflock; - bool initialized; }; #define ATH9K_HTC_TX_CLEANUP_INTERVAL 50 /* ms */ diff --git a/drivers/net/wireless/ath/ath9k/htc_drv_init.c b/drivers/net/wireless/ath/ath9k/htc_drv_init.c index 6de78ae85726..911eaa4f776a 100644 --- a/drivers/net/wireless/ath/ath9k/htc_drv_init.c +++ b/drivers/net/wireless/ath/ath9k/htc_drv_init.c @@ -910,6 +910,9 @@ static int ath9k_init_device(struct ath9k_htc_priv *priv, ath9k_init_leds(priv); ath9k_start_rfkill_poll(priv); + /* signal completion to ath9k_htc_rxep() and ath9k_wmi_event_tasklet() */ + smp_store_release(&priv->initialized, true); + return 0; err_world: @@ -966,10 +969,6 @@ int ath9k_htc_probe_device(struct htc_target *htc_handle, struct device *dev, htc_handle->drv_priv = priv; - /* Allow ath9k_wmi_event_tasklet() to operate. */ - smp_wmb(); - priv->initialized = true; - return 0; err_init: diff --git a/drivers/net/wireless/ath/ath9k/htc_drv_txrx.c b/drivers/net/wireless/ath/ath9k/htc_drv_txrx.c index bed7ea2425a0..95f48c7dca15 100644 --- a/drivers/net/wireless/ath/ath9k/htc_drv_txrx.c +++ b/drivers/net/wireless/ath/ath9k/htc_drv_txrx.c @@ -1141,8 +1141,8 @@ void ath9k_htc_rxep(void *drv_priv, struct sk_buff *skb, struct ath9k_htc_rxbuf *rxbuf = NULL, *tmp_buf = NULL; unsigned long flags; - /* Check if ath9k_rx_init() completed. */ - if (!data_race(priv->rx.initialized)) + /* Check if ath9k_init_device() completed. */ + if (!smp_load_acquire(&priv->initialized)) goto err; spin_lock_irqsave(&priv->rx.rxbuflock, flags); @@ -1200,10 +1200,6 @@ int ath9k_rx_init(struct ath9k_htc_priv *priv) list_add_tail(&rxbuf->list, &priv->rx.rxbuf); } - /* Allow ath9k_htc_rxep() to operate. */ - smp_wmb(); - priv->rx.initialized = true; - return 0; err: diff --git a/drivers/net/wireless/ath/ath9k/wmi.c b/drivers/net/wireless/ath/ath9k/wmi.c index 284e8c13b043..4a08babb6d9d 100644 --- a/drivers/net/wireless/ath/ath9k/wmi.c +++ b/drivers/net/wireless/ath/ath9k/wmi.c @@ -146,6 +146,12 @@ void ath9k_wmi_event_tasklet(struct tasklet_struct *t) unsigned long flags; u16 cmd_id; + /* Check if ath9k_init_device() completed. */ + if (!smp_load_acquire(&priv->initialized)) { + tasklet_schedule(t); + return; + } + do { spin_lock_irqsave(&wmi->wmi_lock, flags); skb = __skb_dequeue(&wmi->wmi_event_queue); @@ -155,12 +161,6 @@ void ath9k_wmi_event_tasklet(struct tasklet_struct *t) } spin_unlock_irqrestore(&wmi->wmi_lock, flags); - /* Check if ath9k_htc_probe_device() completed. */ - if (!data_race(priv->initialized)) { - kfree_skb(skb); - continue; - } - hdr = (struct wmi_cmd_hdr *) skb->data; cmd_id = be16_to_cpu(hdr->command_id); wmi_event = skb_pull(skb, sizeof(struct wmi_cmd_hdr)); @@ -206,9 +206,16 @@ static void ath9k_wmi_rsp_callback(struct wmi *wmi, struct sk_buff *skb) { skb_pull(skb, sizeof(struct wmi_cmd_hdr)); - if (wmi->cmd_rsp_buf != NULL && wmi->cmd_rsp_len != 0) + if (wmi->cmd_rsp_buf && wmi->cmd_rsp_len) { + if (skb->len < wmi->cmd_rsp_len) { + wmi->cmd_rsp_status = -EMSGSIZE; + goto complete; + } memcpy(wmi->cmd_rsp_buf, skb->data, wmi->cmd_rsp_len); + } + wmi->cmd_rsp_status = 0; +complete: complete(&wmi->cmd_wait); } @@ -300,6 +307,7 @@ static int ath9k_wmi_cmd_issue(struct wmi *wmi, /* record the rsp buffer and length */ wmi->cmd_rsp_buf = rsp_buf; wmi->cmd_rsp_len = rsp_len; + wmi->cmd_rsp_status = 0; wmi->last_seq_id = wmi->tx_seq_id; spin_unlock_irqrestore(&wmi->wmi_lock, flags); @@ -356,9 +364,13 @@ int ath9k_wmi_cmd(struct wmi *wmi, enum wmi_cmd_id cmd_id, return -ETIMEDOUT; } + spin_lock_irqsave(&wmi->wmi_lock, flags); + ret = wmi->cmd_rsp_status; + spin_unlock_irqrestore(&wmi->wmi_lock, flags); + mutex_unlock(&wmi->op_mutex); - return 0; + return ret; out: ath_dbg(common, WMI, "WMI failure for: %s\n", wmi_cmd_to_name(cmd_id)); diff --git a/drivers/net/wireless/ath/ath9k/wmi.h b/drivers/net/wireless/ath/ath9k/wmi.h index 5c3b710b8f31..0bb9d31f1a1c 100644 --- a/drivers/net/wireless/ath/ath9k/wmi.h +++ b/drivers/net/wireless/ath/ath9k/wmi.h @@ -157,6 +157,7 @@ struct wmi { u16 tx_seq_id; u8 *cmd_rsp_buf; u32 cmd_rsp_len; + int cmd_rsp_status; bool stopped; struct list_head pending_tx_events; diff --git a/drivers/net/wireless/intel/iwlegacy/3945-mac.c b/drivers/net/wireless/intel/iwlegacy/3945-mac.c index cbaf250626c5..f7ca182d1494 100644 --- a/drivers/net/wireless/intel/iwlegacy/3945-mac.c +++ b/drivers/net/wireless/intel/iwlegacy/3945-mac.c @@ -3807,6 +3807,7 @@ il3945_pci_remove(struct pci_dev *pdev) il_free_channel_map(il); il_free_geos(il); + il_eeprom_free(il); kfree(il->scan_cmd); dev_kfree_skb(il->beacon_skb); ieee80211_free_hw(il->hw); diff --git a/drivers/net/wireless/intersil/p54/fwio.c b/drivers/net/wireless/intersil/p54/fwio.c index a3d9053f043c..f028a7bfa734 100644 --- a/drivers/net/wireless/intersil/p54/fwio.c +++ b/drivers/net/wireless/intersil/p54/fwio.c @@ -52,6 +52,13 @@ int p54_parse_firmware(struct ieee80211_hw *dev, const struct firmware *fw) u32 code = le32_to_cpu(bootrec->code); switch (code) { case BR_CODE_COMPONENT_ID: + if (len < sizeof(struct bootrec_comp_id) / + sizeof(*bootrec->data)) { + wiphy_err(priv->hw->wiphy, + "firmware component ID is too short\n"); + return -EINVAL; + } + priv->fw_interface = be32_to_cpup((__be32 *) bootrec->data); switch (priv->fw_interface) { @@ -72,16 +79,41 @@ int p54_parse_firmware(struct ieee80211_hw *dev, const struct firmware *fw) } break; case BR_CODE_COMPONENT_VERSION: + if (len < DIV_ROUND_UP(sizeof(struct bootrec_comp_ver), + sizeof(*bootrec->data))) { + wiphy_err(priv->hw->wiphy, + "firmware component version is too short\n"); + return -EINVAL; + } + /* 24 bytes should be enough for all firmwares */ - if (strnlen((unsigned char *) bootrec->data, 24) < 24) - fw_version = (unsigned char *) bootrec->data; + if (strnlen((unsigned char *)bootrec->data, + sizeof(struct bootrec_comp_ver)) < + sizeof(struct bootrec_comp_ver)) + fw_version = (unsigned char *)bootrec->data; break; case BR_CODE_DESCR: { struct bootrec_desc *desc = (struct bootrec_desc *)bootrec->data; - priv->rx_start = le32_to_cpu(desc->rx_start); - /* FIXME add sanity checking */ - priv->rx_end = le32_to_cpu(desc->rx_end) - 0x3500; + u32 rx_start, rx_end; + + /* 0xa is the shortest descriptor in supported firmware. */ + if (len < 0xa) { + wiphy_err(priv->hw->wiphy, + "firmware descriptor is too short\n"); + return -EINVAL; + } + + rx_start = le32_to_cpu(desc->rx_start); + rx_end = le32_to_cpu(desc->rx_end); + if (rx_end < 0x3500 || rx_end - 0x3500 <= rx_start) { + wiphy_err(priv->hw->wiphy, + "firmware descriptor has invalid RX range\n"); + return -EINVAL; + } + + priv->rx_start = rx_start; + priv->rx_end = rx_end - 0x3500; priv->headroom = desc->headroom; priv->tailroom = desc->tailroom; priv->privacy_caps = desc->privacy_caps; diff --git a/drivers/net/wireless/silabs/wfx/hif_tx.c b/drivers/net/wireless/silabs/wfx/hif_tx.c index 9f403d275cb1..9efb108bd877 100644 --- a/drivers/net/wireless/silabs/wfx/hif_tx.c +++ b/drivers/net/wireless/silabs/wfx/hif_tx.c @@ -207,9 +207,11 @@ int wfx_hif_read_mib(struct wfx_dev *wdev, int vif_id, u16 mib_id, void *val, si dev_warn(wdev->dev, "%s: confirmation mismatch request\n", __func__); ret = -EIO; } - if (ret == -ENOMEM) + if (!ret && le16_to_cpu(reply->length) > val_len) { dev_err(wdev->dev, "buffer is too small to receive %s (%zu < %d)\n", wfx_get_mib_name(mib_id), val_len, le16_to_cpu(reply->length)); + ret = -EIO; + } if (!ret) memcpy(val, &reply->mib_data, le16_to_cpu(reply->length)); else diff --git a/drivers/net/wireless/st/cw1200/bh.c b/drivers/net/wireless/st/cw1200/bh.c index b034bab4b489..9367a981f1de 100644 --- a/drivers/net/wireless/st/cw1200/bh.c +++ b/drivers/net/wireless/st/cw1200/bh.c @@ -344,6 +344,7 @@ static int cw1200_bh_tx_helper(struct cw1200_common *priv, int *tx_burst) { size_t tx_len; + size_t aligned_len; u8 *data; int ret; struct wsm_hdr *wsm; @@ -376,9 +377,14 @@ static int cw1200_bh_tx_helper(struct cw1200_common *priv, atomic_inc(&priv->bh_tx); - tx_len = priv->hwbus_ops->align_size( + aligned_len = priv->hwbus_ops->align_size( priv->hwbus_priv, tx_len); + /* Zero the bus alignment padding so no stale data is sent. */ + if (aligned_len > tx_len) + memset(data + tx_len, 0, aligned_len - tx_len); + tx_len = aligned_len; + /* Check if not exceeding CW1200 capabilities */ if (WARN_ON_ONCE(tx_len > EFFECTIVE_BUF_SIZE)) pr_debug("Write aligned len: %zu\n", tx_len); diff --git a/drivers/net/wireless/st/cw1200/txrx.c b/drivers/net/wireless/st/cw1200/txrx.c index 084d52b11f5b..e13695667a00 100644 --- a/drivers/net/wireless/st/cw1200/txrx.c +++ b/drivers/net/wireless/st/cw1200/txrx.c @@ -14,6 +14,7 @@ #include "wsm.h" #include "bh.h" #include "sta.h" +#include "hwbus.h" #include "debug.h" #define CW1200_INVALID_RATE_ID (0xFF) @@ -716,6 +717,7 @@ void cw1200_tx(struct ieee80211_hw *dev, struct ieee80211_sta *sta; struct wsm_tx *wsm; bool tid_update = false; + size_t pad_len; u8 flags = 0; int ret; @@ -764,6 +766,18 @@ void cw1200_tx(struct ieee80211_hw *dev, sta = t.sta; + /* + * The bus transfer is padded up to the bus alignment and the + * padding is transferred from the frame buffer. Make sure the + * buffer has room for the padding so the device never receives + * data past the end of the frame. + */ + pad_len = priv->hwbus_ops->align_size(priv->hwbus_priv, skb->len) - + skb->len; + if (skb_tailroom(skb) < pad_len && + pskb_expand_head(skb, 0, pad_len, GFP_ATOMIC)) + goto drop; + spin_lock_bh(&priv->ps_state_lock); { tid_update = cw1200_tx_h_pm_state(priv, &t); diff --git a/drivers/net/wireless/ti/wlcore/main.c b/drivers/net/wireless/ti/wlcore/main.c index edf6ca23c6c3..26aabf4ea127 100644 --- a/drivers/net/wireless/ti/wlcore/main.c +++ b/drivers/net/wireless/ti/wlcore/main.c @@ -6809,8 +6809,10 @@ void wlcore_remove(struct platform_device *pdev) if (pdev_data->family && pdev_data->family->nvs_name) wait_for_completion(&wl->nvs_loading_complete); - if (!wl->initialized) + if (!wl->initialized) { + pm_runtime_put_noidle(wl->dev); return; + } if (wl->wakeirq >= 0) { dev_pm_clear_wake_irq(wl->dev); diff --git a/drivers/nfc/microread/microread.c b/drivers/nfc/microread/microread.c index dfa2490db545..2bfafa94e83d 100644 --- a/drivers/nfc/microread/microread.c +++ b/drivers/nfc/microread/microread.c @@ -251,9 +251,9 @@ static int microread_start_poll(struct nfc_hci_dev *hdev, param[1] |= (1 << 1); if ((im_protocols | tm_protocols) & NFC_PROTO_NFC_DEP_MASK) { - hdev->gb = nfc_get_local_general_bytes(hdev->ndev, - &hdev->gb_len); - if (hdev->gb == NULL || hdev->gb_len == 0) { + nfc_get_local_general_bytes(hdev->ndev, hdev->gb, + sizeof(hdev->gb), &hdev->gb_len); + if (hdev->gb_len == 0) { im_protocols &= ~NFC_PROTO_NFC_DEP_MASK; tm_protocols &= ~NFC_PROTO_NFC_DEP_MASK; } diff --git a/drivers/nfc/nfcmrvl/fw_dnld.c b/drivers/nfc/nfcmrvl/fw_dnld.c index 2b8f401d8fd7..8b9d5257320d 100644 --- a/drivers/nfc/nfcmrvl/fw_dnld.c +++ b/drivers/nfc/nfcmrvl/fw_dnld.c @@ -263,9 +263,14 @@ static int process_state_fw_dnld(struct nfcmrvl_private *priv, * B8..N: payload */ - /* Remove NCI HDR */ - skb_pull(skb, 3); - if (skb->data[0] != HELPER_CMD_PACKET_FORMAT || skb->len != 5) { + if (skb->len != NCI_DATA_HDR_SIZE + 5) { + nfc_err(priv->dev, "bad command"); + return -EINVAL; + } + + /* Remove NCI header */ + skb_pull(skb, NCI_DATA_HDR_SIZE); + if (skb->data[0] != HELPER_CMD_PACKET_FORMAT) { nfc_err(priv->dev, "bad command"); return -EINVAL; } diff --git a/drivers/nfc/pn533/pn533.c b/drivers/nfc/pn533/pn533.c index f5a6a7c20d5a..b0133e51dce9 100644 --- a/drivers/nfc/pn533/pn533.c +++ b/drivers/nfc/pn533/pn533.c @@ -1357,10 +1357,11 @@ static int pn533_poll_dep(struct nfc_dev *nfc_dev) u8 *next, nfcid3[NFC_NFCID3_MAXSIZE]; u8 passive_data[PASSIVE_DATA_LEN] = {0x00, 0xff, 0xff, 0x00, 0x3}; - if (!dev->gb) { - dev->gb = nfc_get_local_general_bytes(nfc_dev, &dev->gb_len); - - if (!dev->gb || !dev->gb_len) { + if (!dev->gb_len) { + nfc_get_local_general_bytes(nfc_dev, dev->gb, + sizeof(dev->gb), + &dev->gb_len); + if (!dev->gb_len) { dev->poll_dep = 0; queue_work(dev->wq, &dev->rf_work); } @@ -1658,8 +1659,9 @@ static int pn533_start_poll(struct nfc_dev *nfc_dev, } if (tm_protocols) { - dev->gb = nfc_get_local_general_bytes(nfc_dev, &dev->gb_len); - if (dev->gb == NULL) + nfc_get_local_general_bytes(nfc_dev, dev->gb, + sizeof(dev->gb), &dev->gb_len); + if (dev->gb_len == 0) tm_protocols = 0; } diff --git a/drivers/nfc/pn533/pn533.h b/drivers/nfc/pn533/pn533.h index 09e35b8693f5..5ab668e05121 100644 --- a/drivers/nfc/pn533/pn533.h +++ b/drivers/nfc/pn533/pn533.h @@ -6,6 +6,8 @@ * Copyright (C) 2012-2013 Tieto Poland */ +#include + #define PN533_DEVICE_STD 0x1 #define PN533_DEVICE_PASORI 0x2 #define PN533_DEVICE_ACR122U 0x3 @@ -166,7 +168,7 @@ struct pn533 { struct timer_list listen_timer; int cancel_listen; - u8 *gb; + u8 gb[NFC_MAX_GT_LEN]; size_t gb_len; u8 tgt_available_prots; diff --git a/drivers/nfc/pn533/usb.c b/drivers/nfc/pn533/usb.c index efb07f944fce..972eaac09e59 100644 --- a/drivers/nfc/pn533/usb.c +++ b/drivers/nfc/pn533/usb.c @@ -319,7 +319,9 @@ static bool pn533_acr122_is_rx_frame_valid(void *_frame, struct pn533 *dev) if (frame->ccid.type != 0x83) return false; - if (!frame->ccid.datalen) + if (frame->ccid.datalen < 2 || + frame->ccid.datalen > PN533_ACR122_FRAME_MAX_PAYLOAD_LEN + + PN533_ACR122_RX_FRAME_TAIL_LEN) return false; if (frame->data[frame->ccid.datalen - 2] == 0x63) diff --git a/drivers/nfc/pn544/pn544.c b/drivers/nfc/pn544/pn544.c index 9d0a16ac465e..c4fa70e45c14 100644 --- a/drivers/nfc/pn544/pn544.c +++ b/drivers/nfc/pn544/pn544.c @@ -377,10 +377,9 @@ static int pn544_hci_start_poll(struct nfc_hci_dev *hdev, return r; if ((im_protocols | tm_protocols) & NFC_PROTO_NFC_DEP_MASK) { - hdev->gb = nfc_get_local_general_bytes(hdev->ndev, - &hdev->gb_len); - pr_debug("generate local bytes %p\n", hdev->gb); - if (hdev->gb == NULL || hdev->gb_len == 0) { + nfc_get_local_general_bytes(hdev->ndev, hdev->gb, + sizeof(hdev->gb), &hdev->gb_len); + if (hdev->gb_len == 0) { im_protocols &= ~NFC_PROTO_NFC_DEP_MASK; tm_protocols &= ~NFC_PROTO_NFC_DEP_MASK; } diff --git a/drivers/nfc/port100.c b/drivers/nfc/port100.c index b613f5e2fd57..e769a30b8b5c 100644 --- a/drivers/nfc/port100.c +++ b/drivers/nfc/port100.c @@ -636,6 +636,13 @@ static void port100_recv_response(struct urb *urb) in_frame = dev->in_urb->transfer_buffer; + if (urb->actual_length < PORT100_FRAME_HEADER_LEN || + urb->actual_length < port100_rx_frame_size(in_frame)) { + nfc_err(&dev->interface->dev, "Received a truncated frame\n"); + cmd->status = -EIO; + goto sched_wq; + } + if (!port100_rx_frame_is_valid(in_frame)) { nfc_err(&dev->interface->dev, "Received an invalid frame\n"); cmd->status = -EIO; diff --git a/drivers/nfc/st21nfca/core.c b/drivers/nfc/st21nfca/core.c index fd39a05c9622..b5c1ca3acfbe 100644 --- a/drivers/nfc/st21nfca/core.c +++ b/drivers/nfc/st21nfca/core.c @@ -351,10 +351,10 @@ static int st21nfca_hci_start_poll(struct nfc_hci_dev *hdev, if (r < 0) return r; } else { - hdev->gb = nfc_get_local_general_bytes(hdev->ndev, - &hdev->gb_len); - - if (hdev->gb == NULL || hdev->gb_len == 0) { + nfc_get_local_general_bytes(hdev->ndev, hdev->gb, + sizeof(hdev->gb), + &hdev->gb_len); + if (hdev->gb_len == 0) { im_protocols &= ~NFC_PROTO_NFC_DEP_MASK; tm_protocols &= ~NFC_PROTO_NFC_DEP_MASK; } @@ -577,9 +577,7 @@ static int st21nfca_get_iso15693_inventory(struct nfc_hci_dev *hdev, if (r < 0) goto exit; - skb_pull(inventory_skb, 2); - - if (inventory_skb->len == 0 || + if (!skb_pull(inventory_skb, 2) || inventory_skb->len < 2 || inventory_skb->len > NFC_ISO15693_UID_MAXSIZE) { r = -EPROTO; goto exit; diff --git a/drivers/nfc/st21nfca/i2c.c b/drivers/nfc/st21nfca/i2c.c index a4c93ff7c5b0..0f44c783bd04 100644 --- a/drivers/nfc/st21nfca/i2c.c +++ b/drivers/nfc/st21nfca/i2c.c @@ -289,27 +289,36 @@ static int check_crc(u8 *buf, int buflen) */ static int st21nfca_hci_i2c_repack(struct sk_buff *skb) { - int i, j, r, size; + int read, write, r, size; - if (skb->len < 1 || (skb->len > 1 && skb->data[1] != 0)) + if (skb->len < ST21NFCA_FRAME_HEADROOM || + !IS_START_OF_FRAME(skb->data)) return -EBADMSG; size = get_frame_size(skb->data, skb->len); if (size > 0) { + if (size < ST21NFCA_FRAME_HEADROOM + 2) + return -EBADMSG; + skb_trim(skb, size); /* remove ST21NFCA byte stuffing for upper layer */ - for (i = 1, j = 0; i < skb->len; i++) { - if (skb->data[i + j] == + for (read = 1, write = 1; read < skb->len;) { + if (skb->data[read] == (u8) ST21NFCA_ESCAPE_BYTE_STUFFING) { - skb->data[i] = skb->data[i + j + 1] - | ST21NFCA_BYTE_STUFFING_MASK; - i++; - j++; + if (read + 1 == skb->len) + return -EBADMSG; + + skb->data[write++] = skb->data[read + 1] + | ST21NFCA_BYTE_STUFFING_MASK; + read += 2; + } else { + skb->data[write++] = skb->data[read++]; } - skb->data[i] = skb->data[i + j]; } /* remove byte stuffing useless byte */ - skb_trim(skb, i - j); + skb_trim(skb, write); + if (skb->len < ST21NFCA_FRAME_HEADROOM + 2) + return -EBADMSG; /* remove ST21NFCA_SOF_EOF from head */ skb_pull(skb, 1); diff --git a/drivers/nfc/trf7970a.c b/drivers/nfc/trf7970a.c index 60883001fa5d..ddfc58c29964 100644 --- a/drivers/nfc/trf7970a.c +++ b/drivers/nfc/trf7970a.c @@ -1997,8 +1997,10 @@ static int trf7970a_startup(struct trf7970a *trf) return ret; ret = trf7970a_update_rx_gain_reduction(trf); - if (ret) + if (ret) { + trf7970a_power_down(trf); return ret; + } pm_runtime_set_active(trf->dev); pm_runtime_enable(trf->dev); diff --git a/drivers/nfc/virtual_ncidev.c b/drivers/nfc/virtual_ncidev.c index 8eeb447ac96e..e51c647b27eb 100644 --- a/drivers/nfc/virtual_ncidev.c +++ b/drivers/nfc/virtual_ncidev.c @@ -195,7 +195,8 @@ static const struct file_operations virtual_ncidev_fops = { .write = virtual_ncidev_write, .open = virtual_ncidev_open, .release = virtual_ncidev_close, - .unlocked_ioctl = virtual_ncidev_ioctl + .unlocked_ioctl = virtual_ncidev_ioctl, + .compat_ioctl = compat_ptr_ioctl, }; static struct miscdevice miscdev = { diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index beea23d04a70..9bcab3dc4c11 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -598,7 +598,6 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl, break; case NVME_CTRL_RESETTING: switch (old_state) { - case NVME_CTRL_NEW: case NVME_CTRL_LIVE: changed = true; atomic_long_inc(&ctrl->nr_reset); @@ -3195,9 +3194,16 @@ static void nvme_release_subsystem(struct device *dev) { struct nvme_subsystem *subsys = container_of(dev, struct nvme_subsystem, dev); + struct nvme_effects_log *cel; + unsigned long i; if (subsys->instance >= 0) ida_free(&nvme_instance_ida, subsys->instance); + xa_for_each(&subsys->cels, i, cel) { + xa_erase(&subsys->cels, i); + kfree(cel); + } + xa_destroy(&subsys->cels); kfree(subsys); } @@ -3311,6 +3317,7 @@ static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) kref_init(&subsys->ref); INIT_LIST_HEAD(&subsys->ctrls); INIT_LIST_HEAD(&subsys->nsheads); + xa_init(&subsys->cels); nvme_init_subnqn(subsys, ctrl, id); memcpy(subsys->serial, id->sn, sizeof(subsys->serial)); memcpy(subsys->model, id->mn, sizeof(subsys->model)); @@ -3413,7 +3420,7 @@ int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi, static int nvme_get_effects_log(struct nvme_ctrl *ctrl, u8 csi, struct nvme_effects_log **log) { - struct nvme_effects_log *old, *cel = xa_load(&ctrl->cels, csi); + struct nvme_effects_log *old, *cel = xa_load(&ctrl->subsys->cels, csi); int ret; if (cel) @@ -3430,7 +3437,7 @@ static int nvme_get_effects_log(struct nvme_ctrl *ctrl, u8 csi, return ret; } - old = xa_store(&ctrl->cels, csi, cel, GFP_KERNEL); + old = xa_store(&ctrl->subsys->cels, csi, cel, GFP_KERNEL); if (xa_is_err(old)) { kfree(cel); return xa_err(old); @@ -3505,7 +3512,7 @@ static int nvme_init_effects_log(struct nvme_ctrl *ctrl, if (!effects) return -ENOMEM; - old = xa_store(&ctrl->cels, csi, effects, GFP_KERNEL); + old = xa_store(&ctrl->subsys->cels, csi, effects, GFP_KERNEL); if (xa_is_err(old)) { kfree(effects); return xa_err(old); @@ -3551,23 +3558,27 @@ static int nvme_init_effects(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) { int ret = 0; + mutex_lock(&ctrl->subsys->lock); + ctrl->effects = xa_load(&ctrl->subsys->cels, NVME_CSI_NVM); if (ctrl->effects) - return 0; + goto out_unlock; if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) { ret = nvme_get_effects_log(ctrl, NVME_CSI_NVM, &ctrl->effects); if (ret < 0) - return ret; + goto out_unlock; } if (!ctrl->effects) { ret = nvme_init_effects_log(ctrl, NVME_CSI_NVM, &ctrl->effects); if (ret < 0) - return ret; + goto out_unlock; } nvme_init_known_nvm_effects(ctrl); - return 0; +out_unlock: + mutex_unlock(&ctrl->subsys->lock); + return ret; } static int nvme_check_ctrl_fabric_info(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) @@ -3931,9 +3942,9 @@ void nvme_cdev_del(struct cdev *cdev, struct device *cdev_device) put_device(cdev_device); } -int nvme_cdev_add(const char *name, struct cdev *cdev, - struct device *cdev_device, - const struct file_operations *fops, struct module *owner) +int nvme_cdev_add(struct cdev *cdev, struct device *cdev_device, + const struct file_operations *fops, struct module *owner, + int ctrl, int head) { int minor, ret; @@ -3941,7 +3952,7 @@ int nvme_cdev_add(const char *name, struct cdev *cdev, if (minor < 0) return minor; - ret = dev_set_name(cdev_device, name); + ret = dev_set_name(cdev_device, "ng%dn%d", ctrl, head); if (ret) { ida_free(&nvme_ns_chr_minor_ida, minor); return ret; @@ -3982,17 +3993,14 @@ static const struct file_operations nvme_ns_chr_fops = { static void nvme_add_ns_cdev(struct nvme_ns *ns) { - char name[32]; - ns->cdev_device.parent = ns->ctrl->device; - snprintf(name, sizeof(name), "ng%dn%d", ns->ctrl->instance, - ns->head->instance); nvme_get_ns(ns); /* Undone in nvme_cdev_rel() */ - if (nvme_cdev_add(name, &ns->cdev, &ns->cdev_device, - &nvme_ns_chr_fops, ns->ctrl->ops->module)) { - dev_err(ns->ctrl->device, "Unable to create the %s device\n", - name); + if (nvme_cdev_add(&ns->cdev, &ns->cdev_device, + &nvme_ns_chr_fops, ns->ctrl->ops->module, + ns->ctrl->instance, ns->head->instance)) { + dev_err(ns->ctrl->device, "Unable to create the ng%dn%d device\n", + ns->ctrl->instance, ns->head->instance); nvme_put_ns(ns); return; } @@ -5155,19 +5163,6 @@ void nvme_uninit_ctrl(struct nvme_ctrl *ctrl) } EXPORT_SYMBOL_GPL(nvme_uninit_ctrl); -static void nvme_free_cels(struct nvme_ctrl *ctrl) -{ - struct nvme_effects_log *cel; - unsigned long i; - - xa_for_each(&ctrl->cels, i, cel) { - xa_erase(&ctrl->cels, i); - kfree(cel); - } - - xa_destroy(&ctrl->cels); -} - static void nvme_free_ctrl(struct device *dev) { struct nvme_ctrl *ctrl = @@ -5180,7 +5175,6 @@ static void nvme_free_ctrl(struct device *dev) blk_put_queue(ctrl->fabrics_q); if (!subsys || ctrl->instance != subsys->instance) ida_free(&nvme_instance_ida, ctrl->instance); - nvme_free_cels(ctrl); nvme_mpath_uninit(ctrl); cleanup_srcu_struct(&ctrl->srcu); nvme_auth_stop(ctrl); @@ -5226,7 +5220,6 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev, mutex_init(&ctrl->scan_lock); INIT_LIST_HEAD(&ctrl->namespaces); - xa_init(&ctrl->cels); ctrl->dev = dev; ctrl->ops = ops; ctrl->quirks = quirks; diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index 48454cb7a0fc..3a4030bcfab7 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -3338,7 +3338,7 @@ nvme_fc_reconnect_or_delete(struct nvme_fc_ctrl *ctrl, int status) ctrl->cnum, min_t(int, portptr->dev_loss_tmo, (ctrl->ctrl.opts->max_reconnects * ctrl->ctrl.opts->reconnect_delay))); - WARN_ON(nvme_delete_ctrl(&ctrl->ctrl)); + nvme_delete_ctrl(&ctrl->ctrl); } } diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index 75dbb58286a3..99ecc73393e5 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -646,17 +646,15 @@ static const struct file_operations nvme_ns_head_chr_fops = { static void nvme_add_ns_head_cdev(struct nvme_ns_head *head) { - char name[32]; - head->cdev_device.parent = &head->subsys->dev; - snprintf(name, sizeof(name), "ng%dn%d", head->subsys->instance, - head->instance); nvme_get_ns_head(head); /* Undone in nvme_cdev_rel() */ - if (nvme_cdev_add(name, &head->cdev, &head->cdev_device, - &nvme_ns_head_chr_fops, THIS_MODULE)) { + if (nvme_cdev_add(&head->cdev, &head->cdev_device, + &nvme_ns_head_chr_fops, THIS_MODULE, + head->subsys->instance, head->instance)) { dev_err(disk_to_dev(head->disk), - "Unable to create the %s device\n", name); + "Unable to create the ng%dn%d device\n", + head->subsys->instance, head->instance); nvme_put_ns_head(head); return; } @@ -763,8 +761,6 @@ int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head) lim.dma_alignment = 3; lim.features |= BLK_FEAT_IO_STAT | BLK_FEAT_NOWAIT | BLK_FEAT_POLL | BLK_FEAT_ATOMIC_WRITES | BLK_FEAT_PCI_P2PDMA; - if (head->ids.csi == NVME_CSI_ZNS) - lim.features |= BLK_FEAT_ZONED; head->disk = blk_alloc_disk(&lim, ctrl->numa_node); if (IS_ERR(head->disk)) @@ -845,7 +841,8 @@ static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data, u32 nr_nsids; size_t nsid_buf_size; - if (WARN_ON_ONCE(offset > ctrl->ana_log_size - sizeof(*desc))) + if (WARN_ON_ONCE(offset > ctrl->ana_log_size || + sizeof(*desc) > ctrl->ana_log_size - offset)) return -EINVAL; nr_nsids = le32_to_cpu(desc->nnsids); @@ -861,7 +858,7 @@ static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data, return -EINVAL; offset += sizeof(*desc); - if (WARN_ON_ONCE(offset > ctrl->ana_log_size - nsid_buf_size)) + if (WARN_ON_ONCE(nsid_buf_size > ctrl->ana_log_size - offset)) return -EINVAL; error = cb(ctrl, desc, data); diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 2cff9fcbf740..a643c4723a69 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -415,7 +415,6 @@ struct nvme_ctrl { unsigned long quirks; struct nvme_id_power_state psd[32]; struct nvme_effects_log *effects; - struct xarray cels; struct work_struct scan_work; struct work_struct async_event_work; struct delayed_work ka_work; @@ -509,6 +508,8 @@ struct nvme_subsystem { __guarded_by(&nvme_subsystems_lock); struct list_head nsheads __guarded_by(&lock); + /* Command effects logs, indexed by CSI and protected by lock. */ + struct xarray cels; char subnqn[NVMF_NQN_SIZE]; char serial[20]; char model[40]; @@ -1013,9 +1014,9 @@ int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi, void nvme_get_ns_head(struct nvme_ns_head *head); bool nvme_tryget_ns_head(struct nvme_ns_head *head); void nvme_put_ns_head(struct nvme_ns_head *head); -int nvme_cdev_add(const char *name, struct cdev *cdev, - struct device *cdev_device, - const struct file_operations *fops, struct module *owner); +int nvme_cdev_add(struct cdev *cdev, struct device *cdev_device, + const struct file_operations *fops, struct module *owner, + int ctrl, int head); void nvme_cdev_del(struct cdev *cdev, struct device *cdev_device); int nvme_ioctl(struct block_device *bdev, blk_mode_t mode, unsigned int cmd, unsigned long arg); diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 5440cf18b55b..77317e5d00f9 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -4196,6 +4196,8 @@ static const struct pci_device_id nvme_id_table[] = { .driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN }, { PCI_DEVICE(0x1344, 0x6001), /* Micron Nitro NVMe */ .driver_data = NVME_QUIRK_BOGUS_NID, }, + { PCI_DEVICE(0x1344, 0x6004), /* Micron 4100AT */ + .driver_data = NVME_QUIRK_DMAPOOL_ALIGN_512, }, { PCI_DEVICE(0x1c5c, 0x1504), /* SK Hynix PC400 */ .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, }, { PCI_DEVICE(0x1c5c, 0x174a), /* SK Hynix P31 SSD */ diff --git a/drivers/nvme/host/sysfs.c b/drivers/nvme/host/sysfs.c index 02a2490a9ed7..e1e3dcfd084b 100644 --- a/drivers/nvme/host/sysfs.c +++ b/drivers/nvme/host/sysfs.c @@ -679,7 +679,7 @@ static ssize_t cntrltype_show(struct device *dev, if (ctrl->cntrltype > NVME_CTRL_ADMIN || !type[ctrl->cntrltype]) return sysfs_emit(buf, "reserved\n"); - return sysfs_emit(buf, type[ctrl->cntrltype]); + return sysfs_emit(buf, "%s", type[ctrl->cntrltype]); } static DEVICE_ATTR_RO(cntrltype); @@ -696,7 +696,7 @@ static ssize_t dctype_show(struct device *dev, if (ctrl->dctype > NVME_DCTYPE_CDC || !type[ctrl->dctype]) return sysfs_emit(buf, "reserved\n"); - return sysfs_emit(buf, type[ctrl->dctype]); + return sysfs_emit(buf, "%s", type[ctrl->dctype]); } static DEVICE_ATTR_RO(dctype); diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index 921934028e0b..7eec983c0ead 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -144,11 +144,6 @@ struct nvme_tcp_queue { void (*state_change)(struct sock *); void (*data_ready)(struct sock *); void (*write_space)(struct sock *); - -#ifdef CONFIG_DEBUG_LOCK_ALLOC - struct lock_class_key nvme_tcp_sk_key; - struct lock_class_key nvme_tcp_slock_key; -#endif }; static DEFINE_MUTEX(nvme_tcp_ctrl_mutex); @@ -185,9 +180,12 @@ static int nvme_tcp_try_send(struct nvme_tcp_queue *queue); * a separate class prevents lockdep from conflating nvme-tcp socket use with * user-space socket API use. */ -static void nvme_tcp_reclassify_socket(struct nvme_tcp_queue *queue) +static struct lock_class_key nvme_tcp_sk_key[2]; +static struct lock_class_key nvme_tcp_slock_key[2]; + +static void nvme_tcp_reclassify_socket(struct socket *sock) { - struct sock *sk = queue->sock->sk; + struct sock *sk = sock->sk; if (WARN_ON_ONCE(!sock_allow_reclassification(sk))) return; @@ -195,20 +193,22 @@ static void nvme_tcp_reclassify_socket(struct nvme_tcp_queue *queue) switch (sk->sk_family) { case AF_INET: sock_lock_init_class_and_name(sk, "slock-AF_INET-NVME", - &queue->nvme_tcp_slock_key, + &nvme_tcp_slock_key[0], "sk_lock-AF_INET-NVME", - &queue->nvme_tcp_sk_key); + &nvme_tcp_sk_key[0]); break; case AF_INET6: sock_lock_init_class_and_name(sk, "slock-AF_INET6-NVME", - &queue->nvme_tcp_slock_key, + &nvme_tcp_slock_key[1], "sk_lock-AF_INET6-NVME", - &queue->nvme_tcp_sk_key); + &nvme_tcp_sk_key[1]); break; default: WARN_ON_ONCE(1); } } +#else +static void nvme_tcp_reclassify_socket(struct socket *sock) { } #endif static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl) @@ -1523,11 +1523,6 @@ static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid) mutex_destroy(&queue->send_mutex); mutex_destroy(&queue->queue_lock); mutex_destroy(&queue->pf_cache_lock); - -#ifdef CONFIG_DEBUG_LOCK_ALLOC - lockdep_unregister_key(&queue->nvme_tcp_sk_key); - lockdep_unregister_key(&queue->nvme_tcp_slock_key); -#endif } static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue) @@ -1875,12 +1870,6 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid, sk_net_refcnt_upgrade(queue->sock->sk); -#ifdef CONFIG_DEBUG_LOCK_ALLOC - lockdep_register_key(&queue->nvme_tcp_sk_key); - lockdep_register_key(&queue->nvme_tcp_slock_key); - nvme_tcp_reclassify_socket(queue); -#endif - /* Single syn retry */ tcp_sock_set_syncnt(queue->sock->sk, 1); @@ -1968,6 +1957,8 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid, goto err_init_connect; } + nvme_tcp_reclassify_socket(queue->sock); + ret = nvme_tcp_init_connection(queue); if (ret) goto err_init_connect; @@ -1984,10 +1975,6 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid, /* Use sync variant - see nvme_tcp_free_queue() for explanation */ __fput_sync(queue->sock->file); queue->sock = NULL; -#ifdef CONFIG_DEBUG_LOCK_ALLOC - lockdep_unregister_key(&queue->nvme_tcp_sk_key); - lockdep_unregister_key(&queue->nvme_tcp_slock_key); -#endif err_destroy_mutex: mutex_destroy(&queue->send_mutex); mutex_destroy(&queue->queue_lock); diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index 6286e38436dd..b03b5d1c2dc3 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -534,6 +534,7 @@ static ssize_t nvmet_ns_device_path_store(struct config_item *item, struct nvmet_subsys *subsys = ns->subsys; size_t len; int ret; + char *new_path = NULL; mutex_lock(&subsys->lock); ret = -EBUSY; @@ -545,12 +546,14 @@ static ssize_t nvmet_ns_device_path_store(struct config_item *item, if (!len) goto out_unlock; - kfree(ns->device_path); ret = -ENOMEM; - ns->device_path = kmemdup_nul(page, len, GFP_KERNEL); - if (!ns->device_path) + new_path = kmemdup_nul(page, len, GFP_KERNEL); + if (!new_path) goto out_unlock; + kfree(ns->device_path); + ns->device_path = new_path; + mutex_unlock(&subsys->lock); return count; diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index 43871a8f56ca..8eea0a504308 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -446,20 +446,27 @@ u16 nvmet_req_find_ns(struct nvmet_req *req) { u32 nsid = le32_to_cpu(req->cmd->common.nsid); struct nvmet_subsys *subsys = nvmet_req_subsys(req); + u16 status = NVME_SC_SUCCESS; + rcu_read_lock(); req->ns = xa_load(&subsys->namespaces, nsid); - if (unlikely(!req->ns || !req->ns->enabled)) { + if (unlikely(!req->ns) || + !test_bit(NVMET_NS_IO_LIVE, &req->ns->flags) || + !percpu_ref_tryget_live_rcu(&req->ns->ref)) { req->error_loc = offsetof(struct nvme_common_command, nsid); - if (!req->ns) /* ns doesn't exist! */ - return NVME_SC_INVALID_NS | NVME_STATUS_DNR; + if (!req->ns) { /* ns doesn't exist! */ + status = NVME_SC_INVALID_NS | NVME_STATUS_DNR; + goto unlock; + } /* ns exists but it's disabled */ req->ns = NULL; - return NVME_SC_INTERNAL_PATH_ERROR; + status = NVME_SC_INTERNAL_PATH_ERROR; } +unlock: + rcu_read_unlock(); - percpu_ref_get(&req->ns->ref); - return NVME_SC_SUCCESS; + return status; } static void nvmet_destroy_namespace(struct percpu_ref *ref) @@ -623,6 +630,7 @@ int nvmet_ns_enable(struct nvmet_ns *ns) ns->enabled = true; xa_set_mark(&subsys->namespaces, ns->nsid, NVMET_NS_ENABLED); nvmet_debugfs_ns_setup(ns); + set_bit(NVMET_NS_IO_LIVE, &ns->flags); ret = 0; out_unlock: mutex_unlock(&subsys->lock); @@ -643,11 +651,11 @@ void nvmet_ns_disable(struct nvmet_ns *ns) struct nvmet_subsys *subsys = ns->subsys; struct nvmet_ctrl *ctrl; - mutex_lock(&subsys->lock); - if (!ns->enabled) - goto out_unlock; + if (!test_and_clear_bit(NVMET_NS_IO_LIVE, &ns->flags)) + return; + + mutex_lock(&subsys->lock); - ns->enabled = false; xa_clear_mark(&subsys->namespaces, ns->nsid, NVMET_NS_ENABLED); nvmet_debugfs_ns_free(ns); @@ -675,7 +683,7 @@ void nvmet_ns_disable(struct nvmet_ns *ns) mutex_lock(&subsys->lock); nvmet_ns_changed(subsys, ns->nsid); nvmet_ns_dev_disable(ns); -out_unlock: + ns->enabled = false; mutex_unlock(&subsys->lock); } @@ -1648,6 +1656,8 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args) INIT_DELAYED_WORK(&ctrl->ka_work, nvmet_keep_alive_timer); memcpy(ctrl->hostnqn, args->hostnqn, NVMF_NQN_SIZE); + if (args->hostid) + uuid_copy(&ctrl->hostid, args->hostid); kref_init(&ctrl->ref); ctrl->subsys = subsys; @@ -1706,9 +1716,6 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args) nvmet_start_keep_alive_timer(ctrl); - if (args->hostid) - uuid_copy(&ctrl->hostid, args->hostid); - dhchap_status = nvmet_setup_auth(ctrl, args->sq, false); if (dhchap_status) { pr_err("Failed to setup authentication, dhchap status %u\n", diff --git a/drivers/nvme/target/fabrics-cmd-auth.c b/drivers/nvme/target/fabrics-cmd-auth.c index 92f8a76f10ff..dccdb55a367c 100644 --- a/drivers/nvme/target/fabrics-cmd-auth.c +++ b/drivers/nvme/target/fabrics-cmd-auth.c @@ -427,7 +427,7 @@ static int nvmet_auth_challenge(struct nvmet_req *req, void *d, int al) struct nvmet_ctrl *ctrl = req->sq->ctrl; int ret = 0; int hash_len = nvme_auth_hmac_hash_len(ctrl->shash_id); - int data_size = sizeof(*d) + hash_len; + int data_size = sizeof(*data) + hash_len; if (ctrl->dh_tfm) data_size += ctrl->dh_keysize; diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index dbda55895f4f..162e2fdd848e 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -112,6 +112,8 @@ struct nvmet_ns { bool buffered_io; bool enabled; +#define NVMET_NS_IO_LIVE 0 + unsigned long flags; struct nvmet_subsys *subsys; const char *device_path; diff --git a/drivers/nvme/target/pci-epf.c b/drivers/nvme/target/pci-epf.c index 9c28367435a5..c683450df422 100644 --- a/drivers/nvme/target/pci-epf.c +++ b/drivers/nvme/target/pci-epf.c @@ -978,6 +978,9 @@ nvmet_pci_epf_get_sgl_segment(struct nvmet_pci_epf_ctrl *ctrl, int nr_descs, ret; void *buf; + if (length < sizeof(struct nvme_sgl_desc)) + return NULL; + buf = kmalloc(length, GFP_KERNEL); if (!buf) return NULL; diff --git a/drivers/of/base.c b/drivers/of/base.c index d71ae0449360..f63a9486b130 100644 --- a/drivers/of/base.c +++ b/drivers/of/base.c @@ -96,9 +96,17 @@ static bool __of_node_is_type(const struct device_node *np, const char *type) return !strcmp(match, type); } +static bool of_coreboot_present(void) +{ + struct device_node *np __free(device_node) = + of_find_compatible_node(NULL, NULL, "coreboot"); + + return np; +} + #define EXCLUDED_DEFAULT_CELLS_PLATFORMS ( \ IS_ENABLED(CONFIG_SPARC) || \ - of_find_compatible_node(NULL, NULL, "coreboot") \ + of_coreboot_present() \ ) int of_bus_n_addr_cells(struct device_node *np) diff --git a/drivers/of/irq.c b/drivers/of/irq.c index ec035367c950..ef1ed9743907 100644 --- a/drivers/of/irq.c +++ b/drivers/of/irq.c @@ -754,6 +754,8 @@ void __init of_irq_init(const struct of_device_id *matches) list_for_each_entry_safe(desc, temp_desc, &intc_parent_list, list) { list_del(&desc->list); + of_node_put(desc->interrupt_parent); + of_node_put(desc->dev); kfree(desc); } err: diff --git a/drivers/of/overlay.c b/drivers/of/overlay.c index 08d5351746be..83c56dd9b5d8 100644 --- a/drivers/of/overlay.c +++ b/drivers/of/overlay.c @@ -256,6 +256,9 @@ static struct property *dup_and_fixup_symbol_prop( if (!target_path) return NULL; target_path_len = strlen(target_path); + /* a root target renders as "/"; drop it to avoid "//" results */ + if (target_path_len == 1 && target_path[0] == '/' && path_tail_len) + target_path_len = 0; new_prop = kzalloc_obj(*new_prop); if (!new_prop) @@ -358,12 +361,13 @@ static int add_changeset_property(struct overlay_changeset *ovcs, return -ENOMEM; if (!prop) { - if (!target->in_livetree) { + ret = of_changeset_add_property(&ovcs->cset, target->np, + new_prop); + /* the detached node owns the property until the apply */ + if (!ret && !target->in_livetree) { new_prop->next = target->np->deadprops; target->np->deadprops = new_prop; } - ret = of_changeset_add_property(&ovcs->cset, target->np, - new_prop); } else { ret = of_changeset_update_property(&ovcs->cset, target->np, new_prop); @@ -853,6 +857,10 @@ static int init_overlay_changeset(struct overlay_changeset *ovcs, err_out: pr_err("%s() failed, ret = %d\n", __func__, ret); + /* let free_overlay_changeset() put the fragments set up so far */ + if (ovcs->fragments) + ovcs->count = cnt; + return ret; } @@ -863,7 +871,8 @@ static void free_overlay_changeset(struct overlay_changeset *ovcs) if (ovcs->cset.entries.next) of_changeset_destroy(&ovcs->cset); - if (ovcs->id) { + /* a failed idr_alloc() leaves its negative error in ovcs->id */ + if (ovcs->id > 0) { idr_remove(&ovcs_idr, ovcs->id); list_del(&ovcs->ovcs_list); ovcs->id = 0; diff --git a/drivers/pci/of_property.c b/drivers/pci/of_property.c index 75a358f73e69..1500740cc55d 100644 --- a/drivers/pci/of_property.c +++ b/drivers/pci/of_property.c @@ -95,9 +95,13 @@ static int of_pci_prop_bus_range(struct pci_dev *pdev, struct of_changeset *ocs, struct device_node *np) { - u32 bus_range[] = { pdev->subordinate->busn_res.start, - pdev->subordinate->busn_res.end }; + u32 bus_range[2]; + if (!pdev->subordinate) + return 0; + + bus_range[0] = pdev->subordinate->busn_res.start; + bus_range[1] = pdev->subordinate->busn_res.end; return of_changeset_add_prop_u32_array(ocs, np, "bus-range", bus_range, ARRAY_SIZE(bus_range)); } @@ -220,6 +224,9 @@ static int of_pci_prop_intr_map(struct pci_dev *pdev, struct of_changeset *ocs, int ret; u8 pin; + if (!pdev->subordinate) + return 0; + pnode = pci_device_to_OF_node(pdev->bus->self); if (!pnode) pnode = pci_bus_to_OF_node(pdev->bus); diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index b2879a6be5f8..8cc6a89130b5 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -3769,8 +3769,21 @@ int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask) } root = pcie_find_root_port(dev); - if (!root) + if (!root) { + + /* + * A hypervisor may expose a topology with the root port + * not visible to the guest. If the hypervisor has already + * set AtomicOp Requester Enable in the endpoint, we assume + * it has verified support in the root port, so it is safe + * for the driver to use AtomicOps. + */ + pcie_capability_read_dword(dev, PCI_EXP_DEVCTL2, &ctl2); + if (ctl2 & PCI_EXP_DEVCTL2_ATOMIC_REQ) + return 0; + return -EINVAL; + } pcie_capability_read_dword(root, PCI_EXP_DEVCAP2, &cap); if ((cap & cap_mask) != cap_mask) diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c index e8c94aa1d3c1..ed16ef7c26fa 100644 --- a/drivers/pci/setup-bus.c +++ b/drivers/pci/setup-bus.c @@ -2380,6 +2380,7 @@ int pci_do_resource_release_and_resize(struct pci_dev *pdev, int resno, int size struct resource *res = pci_resource_n(pdev, resno); struct pci_dev_resource *dev_res; struct pci_bus *bus = pdev->bus; + struct pci_dev *bridge = pci_upstream_bridge(pdev); struct resource *b_win, *r; LIST_HEAD(saved); unsigned int i; @@ -2397,6 +2398,8 @@ int pci_do_resource_release_and_resize(struct pci_dev *pdev, int resno, int size if (ret) return ret; + down_read(&pci_bus_sem); + pci_dev_for_each_resource(pdev, r, i) { if (i >= PCI_BRIDGE_RESOURCES) break; @@ -2415,13 +2418,21 @@ int pci_do_resource_release_and_resize(struct pci_dev *pdev, int resno, int size pci_resize_resource_set_size(pdev, resno, size); - if (!bus->self) - goto out; + if (bridge) { + ret = pbus_reassign_bridge_resources(bus, res, &saved); + if (ret) + goto restore; + } else { + /* No bridge window to adjust; let the core reassign the bus. */ + pci_bus_assign_resources(bus); - down_read(&pci_bus_sem); - ret = pbus_reassign_bridge_resources(bus, res, &saved); - if (ret) - goto restore; + list_for_each_entry(dev_res, &saved, list) { + if (!resource_assigned(dev_res->res)) { + ret = -ENOSPC; + goto restore; + } + } + } out: up_read(&pci_bus_sem); diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c index 33ee2be9b386..5378fba916cf 100644 --- a/drivers/perf/arm-cmn.c +++ b/drivers/perf/arm-cmn.c @@ -305,9 +305,9 @@ enum cmn_filter_select { SEL_NONE, SEL_OCCUP1_ID, SEL_CLASS_OCCUP_ID, - SEL_CBUSY_SNTHROTTLE_SEL, SEL_HBT_LBT_SEL, SEL_SN_HOME_SEL, + SEL_CBUSY_SNTHROTTLE_SEL, SEL_SNP_VC_SEL, SEL_ENHANCED_HBT_LBT_SEL, SEL_EVICT_STATE_SEL, @@ -978,10 +978,14 @@ static umode_t arm_cmn_event_attr_is_visible(struct kobject *kobj, _CMN_EVENT_HNS(_model, _name##_all, _event, _sel, 0), \ _CMN_EVENT_HNS(_model, _name##_hbt, _event, _sel, 1), \ _CMN_EVENT_HNS(_model, _name##_lbt, _event, _sel, 2) -#define _CMN_EVENT_HNS_HBT2(_model, _name, _event, _fsel1, f1) \ - _CMN_EVENT_HNS(_model, _name##_all, _event, _fsel1, f1, SEL_HBT_LBT_SEL, 0), \ - _CMN_EVENT_HNS(_model, _name##_hbt, _event, _fsel1, f1, SEL_HBT_LBT_SEL, 1), \ - _CMN_EVENT_HNS(_model, _name##_lbt, _event, _fsel1, f1, SEL_HBT_LBT_SEL, 2) +#define _CMN_EVENT_HNS_EVICT(_model, _name, _event, _fsel1, f1) \ + _CMN_EVENT_HNS(_model, _name##_all, _event, _fsel1, f1, SEL_EVICT_STATE_SEL, 0), \ + _CMN_EVENT_HNS(_model, _name##_eu, _event, _fsel1, f1, SEL_EVICT_STATE_SEL, 1), \ + _CMN_EVENT_HNS(_model, _name##_en, _event, _fsel1, f1, SEL_EVICT_STATE_SEL, 2), \ + _CMN_EVENT_HNS(_model, _name##_su, _event, _fsel1, f1, SEL_EVICT_STATE_SEL, 3), \ + _CMN_EVENT_HNS(_model, _name##_sn, _event, _fsel1, f1, SEL_EVICT_STATE_SEL, 4), \ + _CMN_EVENT_HNS(_model, _name##_mu, _event, _fsel1, f1, SEL_EVICT_STATE_SEL, 5), \ + _CMN_EVENT_HNS(_model, _name##_mn, _event, _fsel1, f1, SEL_EVICT_STATE_SEL, 6) #define CMN_EVENT_HNS_OCC(_model, _name, _event) \ CMN_EVENT_HN_OCC(_model, hns_##_name, CMN_TYPE_HNS, _event), \ @@ -1014,13 +1018,9 @@ static umode_t arm_cmn_event_attr_is_visible(struct kobject *kobj, _CMN_EVENT_HNS(CMNS3R2, _name##_ccglcn, _event, SEL_ENHANCED_HBT_LBT_SEL, 5), \ _CMN_EVENT_HNS(CMNS3R2, _name##_ccgrn, _event, SEL_ENHANCED_HBT_LBT_SEL, 6) #define CMN_EVENT_HNS_EVICT(_model, _name, _event) \ - _CMN_EVENT_HNS_HBT2(_model, _name##_all, _event, SEL_EVICT_STATE_SEL, 0), \ - _CMN_EVENT_HNS_HBT2(_model, _name##_eu, _event, SEL_EVICT_STATE_SEL, 1), \ - _CMN_EVENT_HNS_HBT2(_model, _name##_en, _event, SEL_EVICT_STATE_SEL, 2), \ - _CMN_EVENT_HNS_HBT2(_model, _name##_su, _event, SEL_EVICT_STATE_SEL, 3), \ - _CMN_EVENT_HNS_HBT2(_model, _name##_sn, _event, SEL_EVICT_STATE_SEL, 4), \ - _CMN_EVENT_HNS_HBT2(_model, _name##_mu, _event, SEL_EVICT_STATE_SEL, 5), \ - _CMN_EVENT_HNS_HBT2(_model, _name##_mn, _event, SEL_EVICT_STATE_SEL, 6) + _CMN_EVENT_HNS_EVICT(_model, _name##_all, _event, SEL_HBT_LBT_SEL, 0), \ + _CMN_EVENT_HNS_EVICT(_model, _name##_hbt, _event, SEL_HBT_LBT_SEL, 1), \ + _CMN_EVENT_HNS_EVICT(_model, _name##_lbt, _event, SEL_HBT_LBT_SEL, 2) #define CMN_EVENT_HNSR0_HBT(_name, _event) \ _CMN_EVENT_HNS_HBT(CMN700 | CMNS3R01, _name, _event, SEL_HBT_LBT_SEL) @@ -1860,10 +1860,12 @@ static void arm_cmn_val_add_event(struct arm_cmn *cmn, struct arm_cmn_val *val, val->dtm_count[dtm]++; + if (sel == SEL_EVICT_STATE_SEL) { + val->filter[dtm][sel] = CMN_EVENT_FILTER2(event) + 1; + sel = SEL_HBT_LBT_SEL; + } if (sel) val->filter[dtm][sel] = CMN_EVENT_FILTER(event) + 1; - if (sel == SEL_EVICT_STATE_SEL) - val->filter[dtm][SEL_HBT_LBT_SEL] = CMN_EVENT_FILTER2(event) + 1; if (type != CMN_TYPE_WP) continue; @@ -1914,14 +1916,17 @@ static int arm_cmn_validate_group(struct arm_cmn *cmn, struct perf_event *event) if (val->dtm_count[dtm] == CMN_DTM_NUM_COUNTERS) goto done; + if (sel == SEL_EVICT_STATE_SEL) { + if (val->filter[dtm][sel] && + val->filter[dtm][sel] != CMN_EVENT_FILTER2(event) + 1) + goto done; + sel = SEL_HBT_LBT_SEL; + } + if (sel && val->filter[dtm][sel] && val->filter[dtm][sel] != CMN_EVENT_FILTER(event) + 1) goto done; - if (sel == SEL_EVICT_STATE_SEL && val->filter[dtm][SEL_HBT_LBT_SEL] && - val->filter[dtm][SEL_HBT_LBT_SEL] != CMN_EVENT_FILTER2(event) + 1) - goto done; - if (type != CMN_TYPE_WP) continue; @@ -1950,7 +1955,7 @@ static enum cmn_filter_select arm_cmn_event_filter(const struct arm_cmn *cmn, for (int i = 0; i < ARRAY_SIZE(arm_cmn_event_attrs) - 1; i++) { e = container_of(arm_cmn_event_attrs[i], typeof(*e), attr.attr); if (e->model & model && e->type == type && e->eventid == eventid) - return e->filter[0].sel; + return e->filter[1].sel ?: e->filter[0].sel; } return SEL_NONE; } @@ -2081,16 +2086,21 @@ static void arm_cmn_event_clear(struct arm_cmn *cmn, struct perf_event *event, static int arm_cmn_set_event_filter(struct arm_cmn_node *dn, struct perf_event *event) { enum cmn_filter_select fsel = to_cmn_hw(event)->filter_sel; + bool evict_state = fsel == SEL_EVICT_STATE_SEL; int ret = 0; + if (evict_state) { + ret = arm_cmn_set_event_sel_hi(dn, fsel, CMN_EVENT_FILTER2(event)); + if (ret) + return ret; + fsel = SEL_HBT_LBT_SEL; + } if (fsel) ret = arm_cmn_set_event_sel_hi(dn, fsel, CMN_EVENT_FILTER(event)); - if (fsel == SEL_EVICT_STATE_SEL && !ret) { - ret = arm_cmn_set_event_sel_hi(dn, SEL_HBT_LBT_SEL, CMN_EVENT_FILTER2(event)); - if (ret) - dn->filter[fsel].count--; - } + if (ret && evict_state) + dn->filter[SEL_EVICT_STATE_SEL].count--; + return ret; } diff --git a/drivers/perf/arm_brbe.c b/drivers/perf/arm_brbe.c index ba554e0c846c..254be4da8ae2 100644 --- a/drivers/perf/arm_brbe.c +++ b/drivers/perf/arm_brbe.c @@ -604,7 +604,7 @@ static bool perf_entry_from_brbe_regset(int index, struct perf_branch_entry *ent return false; brbinf = bregs.brbinf; - perf_clear_branch_entry_bitfields(entry); + *entry = (struct perf_branch_entry){ }; if (brbe_record_is_complete(brbinf)) { entry->from = bregs.brbsrc; entry->to = bregs.brbtgt; diff --git a/drivers/pinctrl/meson/pinctrl-meson-s4.c b/drivers/pinctrl/meson/pinctrl-meson-s4.c index 872948699e9f..365dafe457a9 100644 --- a/drivers/pinctrl/meson/pinctrl-meson-s4.c +++ b/drivers/pinctrl/meson/pinctrl-meson-s4.c @@ -854,7 +854,7 @@ static const char * const i2c1_groups[] = { static const char * const i2c2_groups[] = { "i2c2_sda_d", "i2c2_scl_d", "i2c2_sda_h8", "i2c2_scl_h9", - "i2c2_sda_h0", "i2c2_scl_h1l," + "i2c2_sda_h0", "i2c2_scl_h1", }; static const char * const i2c3_groups[] = { diff --git a/drivers/pinctrl/microchip/pinctrl-mpfs-mssio.c b/drivers/pinctrl/microchip/pinctrl-mpfs-mssio.c index ea1026a0d22c..92d38e5336de 100644 --- a/drivers/pinctrl/microchip/pinctrl-mpfs-mssio.c +++ b/drivers/pinctrl/microchip/pinctrl-mpfs-mssio.c @@ -86,7 +86,7 @@ static struct mpfs_pinctrl_bank_voltage mpfs_pinctrl_bank_voltages[8] = { { .uv = 1800000, .val = 4 }, { .uv = 2500000, .val = 6 }, { .uv = 3300000, .val = 8 }, - { .uv = 0, .val = 0x3f }, // pin unused + { .uv = 0, .val = 0xf }, // pin unused }; static int mpfs_pinctrl_get_drive_strength_ma(u32 drive_strength) @@ -156,10 +156,10 @@ static void mpfs_pinctrl_set_bank_voltage(struct mpfs_pinctrl *pctrl, unsigned i u32 val = FIELD_PREP(MPFS_PINCTRL_BANK_VOLTAGE_MASK, bank_voltage); if (pin < MPFS_PINCTRL_BANK2_START) - regmap_assign_bits(pctrl->sysreg_regmap, MPFS_PINCTRL_MSSIO_BANK4_CFG_CR, + regmap_update_bits(pctrl->sysreg_regmap, MPFS_PINCTRL_MSSIO_BANK4_CFG_CR, MPFS_PINCTRL_BANK_VOLTAGE_MASK, val); else - regmap_assign_bits(pctrl->sysreg_regmap, MPFS_PINCTRL_MSSIO_BANK2_CFG_CR, + regmap_update_bits(pctrl->sysreg_regmap, MPFS_PINCTRL_MSSIO_BANK2_CFG_CR, MPFS_PINCTRL_BANK_VOLTAGE_MASK, val); } diff --git a/drivers/pinctrl/pinctrl-generic.c b/drivers/pinctrl/pinctrl-generic.c index fd6bdb74028a..4277c8748513 100644 --- a/drivers/pinctrl/pinctrl-generic.c +++ b/drivers/pinctrl/pinctrl-generic.c @@ -3,8 +3,10 @@ #define pr_fmt(fmt) "generic pinconfig core: " fmt #include +#include #include #include +#include #include #include @@ -196,6 +198,8 @@ static int pinctrl_generic_dt_node_to_map(struct pinctrl_dev *pctldev, int ngroups = 0; int ret; + guard(mutex)(&pctldev->mutex); + *maps = NULL; *num_maps = 0; diff --git a/drivers/pinctrl/pinctrl-single.c b/drivers/pinctrl/pinctrl-single.c index 4d5f85b7e6bb..e0eb7240a985 100644 --- a/drivers/pinctrl/pinctrl-single.c +++ b/drivers/pinctrl/pinctrl-single.c @@ -1628,7 +1628,8 @@ static int pcs_irq_init_chained_handler(struct pcs_device *pcs, &pcs_irqdomain_ops, pcs_soc); if (!pcs->domain) { - irq_set_chained_handler(pcs_soc->irq, NULL); + pcs_irq_free(pcs); + pcs_soc->irq = -1; return -EINVAL; } diff --git a/drivers/pinctrl/qcom/pinctrl-ipq5210.c b/drivers/pinctrl/qcom/pinctrl-ipq5210.c index 827a4ad07a6b..d5b4eb3e7343 100644 --- a/drivers/pinctrl/qcom/pinctrl-ipq5210.c +++ b/drivers/pinctrl/qcom/pinctrl-ipq5210.c @@ -867,6 +867,7 @@ static const struct of_device_id ipq5210_tlmm_of_match[] = { { .compatible = "qcom,ipq5210-tlmm", }, { }, }; +MODULE_DEVICE_TABLE(of, ipq5210_tlmm_of_match); static int ipq5210_tlmm_probe(struct platform_device *pdev) { diff --git a/drivers/pinctrl/qcom/pinctrl-nord.c b/drivers/pinctrl/qcom/pinctrl-nord.c index 7c21306e77ff..7f37f8e819ba 100644 --- a/drivers/pinctrl/qcom/pinctrl-nord.c +++ b/drivers/pinctrl/qcom/pinctrl-nord.c @@ -570,8 +570,10 @@ enum nord_functions { msm_mux_qup0_se5, msm_mux_qup1_se0, msm_mux_qup1_se1, - msm_mux_qup1_se2, - msm_mux_qup1_se3, + msm_mux_qup1_se2_01, + msm_mux_qup1_se2_23, + msm_mux_qup1_se3_01, + msm_mux_qup1_se3_23, msm_mux_qup1_se4, msm_mux_qup1_se5, msm_mux_qup1_se6, @@ -1152,11 +1154,19 @@ static const char *const qup1_se1_groups[] = { "gpio123", "gpio124", "gpio125", "gpio126", }; -static const char *const qup1_se2_groups[] = { - "gpio127", "gpio128", "gpio129", "gpio130", +static const char *const qup1_se2_01_groups[] = { + "gpio127", "gpio128", }; -static const char *const qup1_se3_groups[] = { +static const char *const qup1_se2_23_groups[] = { + "gpio127", "gpio128", +}; + +static const char *const qup1_se3_01_groups[] = { + "gpio129", "gpio130", +}; + +static const char *const qup1_se3_23_groups[] = { "gpio129", "gpio130", }; @@ -1428,8 +1438,10 @@ static const struct pinfunction nord_functions[] = { MSM_PIN_FUNCTION(qup0_se5), MSM_PIN_FUNCTION(qup1_se0), MSM_PIN_FUNCTION(qup1_se1), - MSM_PIN_FUNCTION(qup1_se2), - MSM_PIN_FUNCTION(qup1_se3), + MSM_PIN_FUNCTION(qup1_se2_01), + MSM_PIN_FUNCTION(qup1_se2_23), + MSM_PIN_FUNCTION(qup1_se3_01), + MSM_PIN_FUNCTION(qup1_se3_23), MSM_PIN_FUNCTION(qup1_se4), MSM_PIN_FUNCTION(qup1_se5), MSM_PIN_FUNCTION(qup1_se6), @@ -1633,13 +1645,13 @@ static const struct msm_pingroup nord_groups[] = { _, _, _, _, _, _, _), [126] = PINGROUP(126, qup1_se1, qup1_se0, ccu_i2c_scl, mdp1_vsync_out, _, atest_usb20, ddr_pxi, _, _, _, _), - [127] = PINGROUP(127, qup1_se2, qup1_se2, _, atest_usb21, ddr_pxi, + [127] = PINGROUP(127, qup1_se2_23, qup1_se2_01, _, atest_usb21, ddr_pxi, _, _, _, _, _, _), - [128] = PINGROUP(128, qup1_se2, qup1_se2, _, atest_usb20, ddr_pxi, + [128] = PINGROUP(128, qup1_se2_23, qup1_se2_01, _, atest_usb20, ddr_pxi, _, _, _, _, _, _), - [129] = PINGROUP(129, qup1_se3, qup1_se3, ccu_i2c_sda, mdp1_vsync_out, + [129] = PINGROUP(129, qup1_se3_23, qup1_se3_01, ccu_i2c_sda, mdp1_vsync_out, _, atest_usb21, ddr_pxi, _, _, _, _), - [130] = PINGROUP(130, qup1_se3, qup1_se3, ccu_i2c_scl, mdp1_vsync_out, + [130] = PINGROUP(130, qup1_se3_23, qup1_se3_01, ccu_i2c_scl, mdp1_vsync_out, _, atest_usb20, ddr_pxi, _, _, _, _), [131] = PINGROUP(131, qup1_se4, qup1_se6, ccu_i2c_sda, mdp1_vsync_out, _, atest_usb21, ddr_pxi, _, _, _, _), diff --git a/drivers/pinctrl/sunxi/pinctrl-sun55i-a523-r.c b/drivers/pinctrl/sunxi/pinctrl-sun55i-a523-r.c index 462aa1c4a5fa..e27e4945def2 100644 --- a/drivers/pinctrl/sunxi/pinctrl-sun55i-a523-r.c +++ b/drivers/pinctrl/sunxi/pinctrl-sun55i-a523-r.c @@ -26,7 +26,7 @@ static const u8 a523_r_irq_bank_muxes[SUNXI_PINCTRL_MAX_BANKS] = static struct sunxi_pinctrl_desc a523_r_pinctrl_data = { .irq_banks = ARRAY_SIZE(a523_r_irq_bank_map), .irq_bank_map = a523_r_irq_bank_map, - .io_bias_cfg_variant = BIAS_VOLTAGE_PIO_POW_MODE_SEL, + .io_bias_cfg_variant = BIAS_VOLTAGE_PIO_POW_MODE_CTL_INV, .pin_base = PL_BASE, }; diff --git a/drivers/pinctrl/sunxi/pinctrl-sun55i-a523.c b/drivers/pinctrl/sunxi/pinctrl-sun55i-a523.c index b6f78f1f30ac..88d8acd5bc24 100644 --- a/drivers/pinctrl/sunxi/pinctrl-sun55i-a523.c +++ b/drivers/pinctrl/sunxi/pinctrl-sun55i-a523.c @@ -26,7 +26,7 @@ static const u8 a523_irq_bank_muxes[SUNXI_PINCTRL_MAX_BANKS] = static struct sunxi_pinctrl_desc a523_pinctrl_data = { .irq_banks = ARRAY_SIZE(a523_irq_bank_map), .irq_bank_map = a523_irq_bank_map, - .io_bias_cfg_variant = BIAS_VOLTAGE_PIO_POW_MODE_SEL, + .io_bias_cfg_variant = BIAS_VOLTAGE_PIO_POW_MODE_CTL_INV, }; static int a523_pinctrl_probe(struct platform_device *pdev) diff --git a/drivers/pinctrl/sunxi/pinctrl-sunxi.c b/drivers/pinctrl/sunxi/pinctrl-sunxi.c index 25489beeb312..31cd142ce0f7 100644 --- a/drivers/pinctrl/sunxi/pinctrl-sunxi.c +++ b/drivers/pinctrl/sunxi/pinctrl-sunxi.c @@ -728,6 +728,7 @@ static int sunxi_pinctrl_set_io_bias_cfg(struct sunxi_pinctrl *pctl, { unsigned short bank; unsigned long flags; + bool inverted = false; u32 val, reg; int uV; @@ -766,6 +767,9 @@ static int sunxi_pinctrl_set_io_bias_cfg(struct sunxi_pinctrl *pctl, reg &= ~IO_BIAS_MASK; writel(reg | val, pctl->membase + sunxi_grp_config_reg(pin)); return 0; + case BIAS_VOLTAGE_PIO_POW_MODE_CTL_INV: + inverted = true; + fallthrough; case BIAS_VOLTAGE_PIO_POW_MODE_CTL: val = uV > 1800000 && uV <= 2500000 ? BIT(bank) : 0; @@ -780,6 +784,8 @@ static int sunxi_pinctrl_set_io_bias_cfg(struct sunxi_pinctrl *pctl, fallthrough; case BIAS_VOLTAGE_PIO_POW_MODE_SEL: val = uV <= 1800000 ? 1 : 0; + if (inverted) + val = !val; raw_spin_lock_irqsave(&pctl->lock, flags); reg = readl(pctl->membase + pctl->pow_mod_sel_offset); @@ -837,6 +843,21 @@ static void sunxi_pmx_set(struct pinctrl_dev *pctldev, writel((readl(pctl->membase + reg) & ~mask) | config << shift, pctl->membase + reg); + /* + * A pin muxed to gpio_out directly through a pinmux node bypasses + * sunxi_pinctrl_gpio_set() and drives whatever its output latch + * holds. Now that the pin is in output mode the data register + * reads back the latch, so refresh the shadow to keep such pins + * driving their pre-existing level. + */ + if (config == SUN4I_FUNC_OUTPUT) { + u32 *shadow = &pctl->dat_shadow[pin / PINS_PER_BANK]; + + sunxi_data_reg(pctl, pin, ®, &shift, &mask); + *shadow = (*shadow & ~mask) | + (readl(pctl->membase + reg) & mask); + } + raw_spin_unlock_irqrestore(&pctl->lock, flags); } @@ -1017,21 +1038,29 @@ static int sunxi_pinctrl_gpio_set(struct gpio_chip *chip, unsigned int offset, int value) { struct sunxi_pinctrl *pctl = gpiochip_get_data(chip); - u32 reg, shift, mask, val; + u32 *shadow = &pctl->dat_shadow[offset / PINS_PER_BANK]; + u32 reg, shift, mask; unsigned long flags; sunxi_data_reg(pctl, offset, ®, &shift, &mask); raw_spin_lock_irqsave(&pctl->lock, flags); - val = readl(pctl->membase + reg); - + /* + * Reading the data register returns the pin level, not the output + * latch, for pins muxed as inputs. A read-modify-write based on + * the register would therefore corrupt the latches of input-muxed + * pins in the same bank (e.g. an emulated open-drain I2C line + * released high), making them drive the wrong level once switched + * to output. Base the read-modify-write on a shadow copy of the + * latches instead. + */ if (value) - val |= mask; + *shadow |= mask; else - val &= ~mask; + *shadow &= ~mask; - writel(val, pctl->membase + reg); + writel(*shadow, pctl->membase + reg); raw_spin_unlock_irqrestore(&pctl->lock, flags); @@ -1572,7 +1601,7 @@ int sunxi_pinctrl_init_with_flags(struct platform_device *pdev, struct pinctrl_pin_desc *pins; struct sunxi_pinctrl *pctl; struct pinmux_ops *pmxops; - int i, ret, last_pin, pin_idx; + int i, ret, last_pin, pin_idx, nbanks; struct clk *clk; pctl = devm_kzalloc(&pdev->dev, sizeof(*pctl), GFP_KERNEL); @@ -1610,6 +1639,37 @@ int sunxi_pinctrl_init_with_flags(struct platform_device *pdev, if (!pctl->irq_array) return -ENOMEM; + /* + * The bus clock has to be enabled before the pinctrl device + * registers, as the pin hogs claimed from there access registers. + */ + ret = of_clk_get_parent_count(node); + clk = devm_clk_get_enabled(&pdev->dev, ret == 1 ? NULL : "apb"); + if (IS_ERR(clk)) + return PTR_ERR(clk); + + /* + * Seed the output latch shadow from the hardware so pins the + * bootloader left in output mode keep their state; see + * sunxi_pinctrl_gpio_set() for why a shadow is needed. This must + * happen before the pinctrl device registers, as pin hogs can mux + * pins to gpio_out and thereby update the shadow. + */ + last_pin = pctl->desc->pins[pctl->desc->npins - 1].pin.number; + nbanks = DIV_ROUND_UP(last_pin + 1 - pctl->desc->pin_base, + PINS_PER_BANK); + pctl->dat_shadow = devm_kcalloc(&pdev->dev, nbanks, + sizeof(*pctl->dat_shadow), GFP_KERNEL); + if (!pctl->dat_shadow) + return -ENOMEM; + + for (i = 0; i < nbanks; i++) { + u32 reg, shift, mask; + + sunxi_data_reg(pctl, i * PINS_PER_BANK, ®, &shift, &mask); + pctl->dat_shadow[i] = readl(pctl->membase + reg); + } + ret = sunxi_pinctrl_build_state(pdev); if (ret) { dev_err(&pdev->dev, "dt probe failed: %d\n", ret); @@ -1665,7 +1725,6 @@ int sunxi_pinctrl_init_with_flags(struct platform_device *pdev, if (!pctl->chip) return -ENOMEM; - last_pin = pctl->desc->pins[pctl->desc->npins - 1].pin.number; pctl->chip->owner = THIS_MODULE; pctl->chip->request = gpiochip_generic_request; pctl->chip->free = gpiochip_generic_free; @@ -1699,13 +1758,6 @@ int sunxi_pinctrl_init_with_flags(struct platform_device *pdev, goto gpiochip_error; } - ret = of_clk_get_parent_count(node); - clk = devm_clk_get_enabled(&pdev->dev, ret == 1 ? NULL : "apb"); - if (IS_ERR(clk)) { - ret = PTR_ERR(clk); - goto gpiochip_error; - } - pctl->irq = devm_kcalloc(&pdev->dev, pctl->desc->irq_banks, sizeof(*pctl->irq), diff --git a/drivers/pinctrl/sunxi/pinctrl-sunxi.h b/drivers/pinctrl/sunxi/pinctrl-sunxi.h index 0daf7600e2fb..8bd00c6ff628 100644 --- a/drivers/pinctrl/sunxi/pinctrl-sunxi.h +++ b/drivers/pinctrl/sunxi/pinctrl-sunxi.h @@ -85,6 +85,7 @@ #define IO_BIAS_MASK GENMASK(3, 0) #define SUN4I_FUNC_INPUT 0 +#define SUN4I_FUNC_OUTPUT 1 #define SUN4I_FUNC_IRQ 6 #define SUN4I_FUNC_DISABLED_OLD 7 #define SUN4I_FUNC_DISABLED_NEW 15 @@ -116,8 +117,10 @@ enum sunxi_desc_bias_voltage { * Bias voltage is set through PIO_POW_MOD_SEL_REG * and PIO_POW_MOD_CTL_REG register, as seen on * A100 and D1 SoC, for example. + * Some SoCs invert the encoding for 1.8V vs. 3.3V. */ BIAS_VOLTAGE_PIO_POW_MODE_CTL, + BIAS_VOLTAGE_PIO_POW_MODE_CTL_INV, }; struct sunxi_desc_function { @@ -175,6 +178,12 @@ struct sunxi_pinctrl { int *irq; unsigned *irq_array; raw_spinlock_t lock; + /* + * Output latch shadow, one word per bank. Seeded lockless at + * probe before the pinctrl device registers, protected by @lock + * afterwards. + */ + u32 *dat_shadow; struct pinctrl_dev *pctl_dev; unsigned long flags; u32 bank_mem_size; diff --git a/drivers/pinctrl/tegra/pinctrl-tegra238.c b/drivers/pinctrl/tegra/pinctrl-tegra238.c index ec482365f14f..40aba285944e 100644 --- a/drivers/pinctrl/tegra/pinctrl-tegra238.c +++ b/drivers/pinctrl/tegra/pinctrl-tegra238.c @@ -1744,57 +1744,57 @@ static const char * const tegra238_functions[] = { #define drive_sdmmc1_dat0_pu2 DRV_PINGROUP_ENTRY_Y(0x8034, 28, 2, 30, 2, -1, -1, -1, -1, 0) #define drive_ufs0_rst_n_pv1 DRV_PINGROUP_ENTRY_Y(0x11004, 12, 5, 24, 5, -1, -1, -1, -1, 0) #define drive_ufs0_ref_clk_pv0 DRV_PINGROUP_ENTRY_Y(0x1100c, 12, 5, 24, 5, -1, -1, -1, -1, 0) -#define drive_batt_oc_paa4 DRV_PINGROUP_ENTRY_Y(0x1024, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_bootv_ctl_n_paa0 DRV_PINGROUP_ENTRY_Y(0x102c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_vcomp_alert_paa2 DRV_PINGROUP_ENTRY_Y(0x105c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_hdmi_cec_pbb0 DRV_PINGROUP_ENTRY_Y(0x1064, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_touch_clk_pdd3 DRV_PINGROUP_ENTRY_Y(0x106c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_uart3_rx_pcc6 DRV_PINGROUP_ENTRY_Y(0x1074, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_uart3_tx_pcc5 DRV_PINGROUP_ENTRY_Y(0x107c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_gen8_i2c_sda_pdd2 DRV_PINGROUP_ENTRY_Y(0x1084, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_gen8_i2c_scl_pdd1 DRV_PINGROUP_ENTRY_Y(0x108c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_spi2_mosi_pcc2 DRV_PINGROUP_ENTRY_Y(0x1094, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_gen2_i2c_scl_pcc7 DRV_PINGROUP_ENTRY_Y(0x109c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_spi2_cs0_pcc3 DRV_PINGROUP_ENTRY_Y(0x10a4, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_gen2_i2c_sda_pdd0 DRV_PINGROUP_ENTRY_Y(0x10ac, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_spi2_sck_pcc0 DRV_PINGROUP_ENTRY_Y(0x10b4, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_spi2_miso_pcc1 DRV_PINGROUP_ENTRY_Y(0x10bc, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio49_pee2 DRV_PINGROUP_ENTRY_Y(0x10c4, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio50_pee4 DRV_PINGROUP_ENTRY_Y(0x10cc, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio82_pee3 DRV_PINGROUP_ENTRY_Y(0x10d4, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio71_pff2 DRV_PINGROUP_ENTRY_Y(0x10dc, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio76_pff7 DRV_PINGROUP_ENTRY_Y(0x10e4, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio74_pff5 DRV_PINGROUP_ENTRY_Y(0x10ec, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio00_paa1 DRV_PINGROUP_ENTRY_Y(0x10f4, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio19_pdd6 DRV_PINGROUP_ENTRY_Y(0x10fc, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio86_phh3 DRV_PINGROUP_ENTRY_Y(0x1104, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio72_pff3 DRV_PINGROUP_ENTRY_Y(0x110c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio77_pgg0 DRV_PINGROUP_ENTRY_Y(0x1114, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio80_pff6 DRV_PINGROUP_ENTRY_Y(0x111c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio84_pgg1 DRV_PINGROUP_ENTRY_Y(0x1124, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio83_pee5 DRV_PINGROUP_ENTRY_Y(0x112c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio73_pff4 DRV_PINGROUP_ENTRY_Y(0x1134, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio70_pff1 DRV_PINGROUP_ENTRY_Y(0x113c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio04_paa5 DRV_PINGROUP_ENTRY_Y(0x1144, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio85_pgg6 DRV_PINGROUP_ENTRY_Y(0x114c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio69_pff0 DRV_PINGROUP_ENTRY_Y(0x1154, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio25_paa6 DRV_PINGROUP_ENTRY_Y(0x115c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_soc_gpio26_paa7 DRV_PINGROUP_ENTRY_Y(0x1164, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_uart5_tx_pgg7 DRV_PINGROUP_ENTRY_Y(0x116c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_uart5_rx_phh0 DRV_PINGROUP_ENTRY_Y(0x1174, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_uart2_tx_pgg2 DRV_PINGROUP_ENTRY_Y(0x117c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_uart2_rx_pgg3 DRV_PINGROUP_ENTRY_Y(0x1184, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_uart2_cts_pgg5 DRV_PINGROUP_ENTRY_Y(0x118c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_uart2_rts_pgg4 DRV_PINGROUP_ENTRY_Y(0x1194, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_uart5_cts_phh2 DRV_PINGROUP_ENTRY_Y(0x119c, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_uart5_rts_phh1 DRV_PINGROUP_ENTRY_Y(0x11a4, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_pwm7_pee1 DRV_PINGROUP_ENTRY_Y(0x11ac, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_pwm2_pdd7 DRV_PINGROUP_ENTRY_Y(0x11b4, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_pwm3_pee0 DRV_PINGROUP_ENTRY_Y(0x11bc, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_pwm1_paa3 DRV_PINGROUP_ENTRY_Y(0x11c4, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_spi2_cs1_pcc4 DRV_PINGROUP_ENTRY_Y(0x11cc, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_dmic1_clk_pdd4 DRV_PINGROUP_ENTRY_Y(0x11d4, 12, 5, 20, 5, -1, -1, -1, -1, 1) -#define drive_dmic1_dat_pdd5 DRV_PINGROUP_ENTRY_Y(0x11dc, 12, 5, 20, 5, -1, -1, -1, -1, 1) +#define drive_batt_oc_paa4 DRV_PINGROUP_ENTRY_Y(0x1024, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_bootv_ctl_n_paa0 DRV_PINGROUP_ENTRY_Y(0x102c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_vcomp_alert_paa2 DRV_PINGROUP_ENTRY_Y(0x105c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_hdmi_cec_pbb0 DRV_PINGROUP_ENTRY_Y(0x1064, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_touch_clk_pdd3 DRV_PINGROUP_ENTRY_Y(0x106c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_uart3_rx_pcc6 DRV_PINGROUP_ENTRY_Y(0x1074, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_uart3_tx_pcc5 DRV_PINGROUP_ENTRY_Y(0x107c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_gen8_i2c_sda_pdd2 DRV_PINGROUP_ENTRY_Y(0x1084, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_gen8_i2c_scl_pdd1 DRV_PINGROUP_ENTRY_Y(0x108c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_spi2_mosi_pcc2 DRV_PINGROUP_ENTRY_Y(0x1094, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_gen2_i2c_scl_pcc7 DRV_PINGROUP_ENTRY_Y(0x109c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_spi2_cs0_pcc3 DRV_PINGROUP_ENTRY_Y(0x10a4, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_gen2_i2c_sda_pdd0 DRV_PINGROUP_ENTRY_Y(0x10ac, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_spi2_sck_pcc0 DRV_PINGROUP_ENTRY_Y(0x10b4, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_spi2_miso_pcc1 DRV_PINGROUP_ENTRY_Y(0x10bc, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio49_pee2 DRV_PINGROUP_ENTRY_Y(0x10c4, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio50_pee4 DRV_PINGROUP_ENTRY_Y(0x10cc, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio82_pee3 DRV_PINGROUP_ENTRY_Y(0x10d4, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio71_pff2 DRV_PINGROUP_ENTRY_Y(0x10dc, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio76_pff7 DRV_PINGROUP_ENTRY_Y(0x10e4, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio74_pff5 DRV_PINGROUP_ENTRY_Y(0x10ec, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio00_paa1 DRV_PINGROUP_ENTRY_Y(0x10f4, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio19_pdd6 DRV_PINGROUP_ENTRY_Y(0x10fc, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio86_phh3 DRV_PINGROUP_ENTRY_Y(0x1104, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio72_pff3 DRV_PINGROUP_ENTRY_Y(0x110c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio77_pgg0 DRV_PINGROUP_ENTRY_Y(0x1114, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio80_pff6 DRV_PINGROUP_ENTRY_Y(0x111c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio84_pgg1 DRV_PINGROUP_ENTRY_Y(0x1124, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio83_pee5 DRV_PINGROUP_ENTRY_Y(0x112c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio73_pff4 DRV_PINGROUP_ENTRY_Y(0x1134, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio70_pff1 DRV_PINGROUP_ENTRY_Y(0x113c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio04_paa5 DRV_PINGROUP_ENTRY_Y(0x1144, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio85_pgg6 DRV_PINGROUP_ENTRY_Y(0x114c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio69_pff0 DRV_PINGROUP_ENTRY_Y(0x1154, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio25_paa6 DRV_PINGROUP_ENTRY_Y(0x115c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_soc_gpio26_paa7 DRV_PINGROUP_ENTRY_Y(0x1164, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_uart5_tx_pgg7 DRV_PINGROUP_ENTRY_Y(0x116c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_uart5_rx_phh0 DRV_PINGROUP_ENTRY_Y(0x1174, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_uart2_tx_pgg2 DRV_PINGROUP_ENTRY_Y(0x117c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_uart2_rx_pgg3 DRV_PINGROUP_ENTRY_Y(0x1184, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_uart2_cts_pgg5 DRV_PINGROUP_ENTRY_Y(0x118c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_uart2_rts_pgg4 DRV_PINGROUP_ENTRY_Y(0x1194, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_uart5_cts_phh2 DRV_PINGROUP_ENTRY_Y(0x119c, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_uart5_rts_phh1 DRV_PINGROUP_ENTRY_Y(0x11a4, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_pwm7_pee1 DRV_PINGROUP_ENTRY_Y(0x11ac, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_pwm2_pdd7 DRV_PINGROUP_ENTRY_Y(0x11b4, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_pwm3_pee0 DRV_PINGROUP_ENTRY_Y(0x11bc, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_pwm1_paa3 DRV_PINGROUP_ENTRY_Y(0x11c4, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_spi2_cs1_pcc4 DRV_PINGROUP_ENTRY_Y(0x11cc, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_dmic1_clk_pdd4 DRV_PINGROUP_ENTRY_Y(0x11d4, 12, 5, 20, 5, -1, -1, -1, -1, 0) +#define drive_dmic1_dat_pdd5 DRV_PINGROUP_ENTRY_Y(0x11dc, 12, 5, 20, 5, -1, -1, -1, -1, 0) #define drive_sdmmc1_comp DRV_PINGROUP_ENTRY_N @@ -1961,57 +1961,57 @@ static const struct tegra_pingroup tegra238_groups[] = { }; static const struct tegra_pingroup tegra238_aon_groups[] = { - PINGROUP(bootv_ctl_n_paa0, RSVD0, RSVD1, RSVD2, RSVD3, 0x1028, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio00_paa1, RSVD0, RSVD1, RSVD2, RSVD3, 0x10f0, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(vcomp_alert_paa2, SOC_THERM_OC1, RSVD1, RSVD2, RSVD3, 0x1058, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(pwm1_paa3, GP_PWM1, RSVD1, RSVD2, RSVD3, 0x11c0, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(batt_oc_paa4, SOC_THERM_OC2, RSVD1, RSVD2, RSVD3, 0x1020, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio04_paa5, RSVD0, RSVD1, RSVD2, RSVD3, 0x1140, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio25_paa6, RSVD0, RSVD1, RSVD2, RSVD3, 0x1158, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio26_paa7, RSVD0, SOC_THERM_OC3, RSVD2, RSVD3, 0x1160, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(hdmi_cec_pbb0, HDMI_CEC, RSVD1, RSVD2, RSVD3, 0x1060, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(spi2_sck_pcc0, SPI2_SCK, RSVD1, RSVD2, RSVD3, 0x10b0, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(spi2_miso_pcc1, SPI2_DIN, RSVD1, RSVD2, RSVD3, 0x10b8, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(spi2_mosi_pcc2, SPI2_DOUT, RSVD1, RSVD2, RSVD3, 0x1090, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(spi2_cs0_pcc3, SPI2_CS0, RSVD1, RSVD2, RSVD3, 0x10a0, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(spi2_cs1_pcc4, SPI2_CS1, RSVD1, RSVD2, RSVD3, 0x11c8, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(uart3_tx_pcc5, UARTC_TXD, RSVD1, RSVD2, RSVD3, 0x1078, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(uart3_rx_pcc6, UARTC_RXD, RSVD1, RSVD2, RSVD3, 0x1070, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(gen2_i2c_scl_pcc7, I2C2_CLK, RSVD1, RSVD2, RSVD3, 0x1098, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(gen2_i2c_sda_pdd0, I2C2_DAT, RSVD1, RSVD2, RSVD3, 0x10a8, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(gen8_i2c_scl_pdd1, I2C8_CLK, RSVD1, RSVD2, RSVD3, 0x1088, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(gen8_i2c_sda_pdd2, I2C8_DAT, RSVD1, RSVD2, RSVD3, 0x1080, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(touch_clk_pdd3, GP_PWM4, TOUCH_CLK, RSVD2, RSVD3, 0x1068, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(dmic1_clk_pdd4, DMIC1_CLK, RSVD1, DMIC5_CLK, RSVD3, 0x11d0, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(dmic1_dat_pdd5, DMIC1_DAT, RSVD1, DMIC5_DAT, RSVD3, 0x11d8, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio19_pdd6, RSVD0, WDT_RESET_OUTB, RSVD2, RSVD3, 0x10f8, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio49_pee2, RSVD0, RSVD1, RSVD2, RSVD3, 0x10c0, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio50_pee4, RSVD0, RSVD1, RSVD2, RSVD3, 0x10c8, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio82_pee3, RSVD0, RSVD1, RSVD2, RSVD3, 0x10d0, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio71_pff2, PPC_MODE_1, RSVD1, RSVD2, RSVD3, 0x10d8, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio76_pff7, RSVD0, RSVD1, TSC_EDGE_OUT0, TSC_EDGE_OUT0A, 0x10e0, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio74_pff5, PPC_READY, PPC_I2C_DAT, RSVD2, RSVD3, 0x10e8, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio86_phh3, RSVD0, SPI5_CS1, TSC_EDGE_OUT3, TSC_EDGE_OUT0D, 0x1100, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio72_pff3, PPC_MODE_2, RSVD1, RSVD2, RSVD3, 0x1108, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio77_pgg0, RSVD0, RSVD1, TSC_EDGE_OUT1, TSC_EDGE_OUT0B, 0x1110, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio80_pff6, RSVD0, PPC_RST_N, RSVD2, RSVD3, 0x1118, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio84_pgg1, RSVD0, RSVD1, TSC_EDGE_OUT2, TSC_EDGE_OUT0C, 0x1120, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio83_pee5, RSVD0, RSVD1, RSVD2, RSVD3, 0x1128, 1, Y, -1, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio73_pff4, PPC_CC, PPC_I2C_CLK, RSVD2, RSVD3, 0x1130, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio70_pff1, PPC_MODE_0, RSVD1, RSVD2, RSVD3, 0x1138, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio85_pgg6, RSVD0, SPI4_CS1, RSVD2, RSVD3, 0x1148, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(soc_gpio69_pff0, PPC_INT_N, RSVD1, RSVD2, RSVD3, 0x1150, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(uart5_tx_pgg7, UARTE_TXD, SPI5_SCK, RSVD2, RSVD3, 0x1168, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(uart5_rx_phh0, UARTE_RXD, SPI5_MISO, RSVD2, RSVD3, 0x1170, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(uart2_tx_pgg2, UARTB_TXD, SPI4_SCK, RSVD2, RSVD3, 0x1178, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(uart2_rx_pgg3, UARTB_RXD, SPI4_MISO, RSVD2, RSVD3, 0x1180, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(uart2_cts_pgg5, UARTB_CTS, SPI4_CS0, RSVD2, RSVD3, 0x1188, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(uart2_rts_pgg4, UARTB_RTS, SPI4_MOSI, RSVD2, RSVD3, 0x1190, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(uart5_cts_phh2, UARTE_CTS, SPI5_CS0, RSVD2, RSVD3, 0x1198, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(uart5_rts_phh1, UARTE_RTS, SPI5_MOSI, RSVD2, RSVD3, 0x11a0, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(pwm2_pdd7, GP_PWM2, LED_BLINK, RSVD2, RSVD3, 0x11b0, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(pwm3_pee0, GP_PWM3, RSVD1, RSVD2, RSVD3, 0x11b8, 1, Y, 5, 7, 6, 8, -1, 10, 12), - PINGROUP(pwm7_pee1, GP_PWM7, RSVD1, RSVD2, RSVD3, 0x11a8, 1, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(bootv_ctl_n_paa0, RSVD0, RSVD1, RSVD2, RSVD3, 0x1028, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio00_paa1, RSVD0, RSVD1, RSVD2, RSVD3, 0x10f0, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(vcomp_alert_paa2, SOC_THERM_OC1, RSVD1, RSVD2, RSVD3, 0x1058, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(pwm1_paa3, GP_PWM1, RSVD1, RSVD2, RSVD3, 0x11c0, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(batt_oc_paa4, SOC_THERM_OC2, RSVD1, RSVD2, RSVD3, 0x1020, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio04_paa5, RSVD0, RSVD1, RSVD2, RSVD3, 0x1140, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio25_paa6, RSVD0, RSVD1, RSVD2, RSVD3, 0x1158, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio26_paa7, RSVD0, SOC_THERM_OC3, RSVD2, RSVD3, 0x1160, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(hdmi_cec_pbb0, HDMI_CEC, RSVD1, RSVD2, RSVD3, 0x1060, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(spi2_sck_pcc0, SPI2_SCK, RSVD1, RSVD2, RSVD3, 0x10b0, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(spi2_miso_pcc1, SPI2_DIN, RSVD1, RSVD2, RSVD3, 0x10b8, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(spi2_mosi_pcc2, SPI2_DOUT, RSVD1, RSVD2, RSVD3, 0x1090, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(spi2_cs0_pcc3, SPI2_CS0, RSVD1, RSVD2, RSVD3, 0x10a0, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(spi2_cs1_pcc4, SPI2_CS1, RSVD1, RSVD2, RSVD3, 0x11c8, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(uart3_tx_pcc5, UARTC_TXD, RSVD1, RSVD2, RSVD3, 0x1078, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(uart3_rx_pcc6, UARTC_RXD, RSVD1, RSVD2, RSVD3, 0x1070, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(gen2_i2c_scl_pcc7, I2C2_CLK, RSVD1, RSVD2, RSVD3, 0x1098, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(gen2_i2c_sda_pdd0, I2C2_DAT, RSVD1, RSVD2, RSVD3, 0x10a8, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(gen8_i2c_scl_pdd1, I2C8_CLK, RSVD1, RSVD2, RSVD3, 0x1088, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(gen8_i2c_sda_pdd2, I2C8_DAT, RSVD1, RSVD2, RSVD3, 0x1080, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(touch_clk_pdd3, GP_PWM4, TOUCH_CLK, RSVD2, RSVD3, 0x1068, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(dmic1_clk_pdd4, DMIC1_CLK, RSVD1, DMIC5_CLK, RSVD3, 0x11d0, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(dmic1_dat_pdd5, DMIC1_DAT, RSVD1, DMIC5_DAT, RSVD3, 0x11d8, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio19_pdd6, RSVD0, WDT_RESET_OUTB, RSVD2, RSVD3, 0x10f8, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio49_pee2, RSVD0, RSVD1, RSVD2, RSVD3, 0x10c0, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio50_pee4, RSVD0, RSVD1, RSVD2, RSVD3, 0x10c8, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio82_pee3, RSVD0, RSVD1, RSVD2, RSVD3, 0x10d0, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio71_pff2, PPC_MODE_1, RSVD1, RSVD2, RSVD3, 0x10d8, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio76_pff7, RSVD0, RSVD1, TSC_EDGE_OUT0, TSC_EDGE_OUT0A, 0x10e0, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio74_pff5, PPC_READY, PPC_I2C_DAT, RSVD2, RSVD3, 0x10e8, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio86_phh3, RSVD0, SPI5_CS1, TSC_EDGE_OUT3, TSC_EDGE_OUT0D, 0x1100, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio72_pff3, PPC_MODE_2, RSVD1, RSVD2, RSVD3, 0x1108, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio77_pgg0, RSVD0, RSVD1, TSC_EDGE_OUT1, TSC_EDGE_OUT0B, 0x1110, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio80_pff6, RSVD0, PPC_RST_N, RSVD2, RSVD3, 0x1118, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio84_pgg1, RSVD0, RSVD1, TSC_EDGE_OUT2, TSC_EDGE_OUT0C, 0x1120, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio83_pee5, RSVD0, RSVD1, RSVD2, RSVD3, 0x1128, 0, Y, -1, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio73_pff4, PPC_CC, PPC_I2C_CLK, RSVD2, RSVD3, 0x1130, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio70_pff1, PPC_MODE_0, RSVD1, RSVD2, RSVD3, 0x1138, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio85_pgg6, RSVD0, SPI4_CS1, RSVD2, RSVD3, 0x1148, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(soc_gpio69_pff0, PPC_INT_N, RSVD1, RSVD2, RSVD3, 0x1150, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(uart5_tx_pgg7, UARTE_TXD, SPI5_SCK, RSVD2, RSVD3, 0x1168, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(uart5_rx_phh0, UARTE_RXD, SPI5_MISO, RSVD2, RSVD3, 0x1170, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(uart2_tx_pgg2, UARTB_TXD, SPI4_SCK, RSVD2, RSVD3, 0x1178, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(uart2_rx_pgg3, UARTB_RXD, SPI4_MISO, RSVD2, RSVD3, 0x1180, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(uart2_cts_pgg5, UARTB_CTS, SPI4_CS0, RSVD2, RSVD3, 0x1188, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(uart2_rts_pgg4, UARTB_RTS, SPI4_MOSI, RSVD2, RSVD3, 0x1190, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(uart5_cts_phh2, UARTE_CTS, SPI5_CS0, RSVD2, RSVD3, 0x1198, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(uart5_rts_phh1, UARTE_RTS, SPI5_MOSI, RSVD2, RSVD3, 0x11a0, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(pwm2_pdd7, GP_PWM2, LED_BLINK, RSVD2, RSVD3, 0x11b0, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(pwm3_pee0, GP_PWM3, RSVD1, RSVD2, RSVD3, 0x11b8, 0, Y, 5, 7, 6, 8, -1, 10, 12), + PINGROUP(pwm7_pee1, GP_PWM7, RSVD1, RSVD2, RSVD3, 0x11a8, 0, Y, 5, 7, 6, 8, -1, 10, 12), }; static const struct tegra_pinctrl_soc_data tegra238_pinctrl_aon = { diff --git a/drivers/remoteproc/qcom_q6v5_adsp.c b/drivers/remoteproc/qcom_q6v5_adsp.c index 39654206781d..2e5fb5954fa9 100644 --- a/drivers/remoteproc/qcom_q6v5_adsp.c +++ b/drivers/remoteproc/qcom_q6v5_adsp.c @@ -104,6 +104,7 @@ struct qcom_adsp { struct completion stop_done; phys_addr_t mem_phys; + unsigned long iova; phys_addr_t mem_reloc; void __iomem *mem_region; size_t mem_size; @@ -333,7 +334,7 @@ static void adsp_unmap_carveout(struct rproc *rproc) struct qcom_adsp *adsp = rproc->priv; if (adsp->has_iommu) - iommu_unmap(rproc->domain, adsp->mem_phys, adsp->mem_size); + iommu_unmap(rproc->domain, adsp->iova, adsp->mem_size); } static int adsp_map_carveout(struct rproc *rproc) @@ -341,7 +342,6 @@ static int adsp_map_carveout(struct rproc *rproc) struct qcom_adsp *adsp = rproc->priv; struct of_phandle_args args; long long sid; - unsigned long iova; int ret; if (!adsp->has_iommu) @@ -358,9 +358,9 @@ static int adsp_map_carveout(struct rproc *rproc) of_node_put(args.np); /* Add SID configuration for ADSP Firmware to SMMU */ - iova = adsp->mem_phys | (sid << 32); + adsp->iova = adsp->mem_phys | (sid << 32); - ret = iommu_map(rproc->domain, iova, adsp->mem_phys, + ret = iommu_map(rproc->domain, adsp->iova, adsp->mem_phys, adsp->mem_size, IOMMU_READ | IOMMU_WRITE, GFP_KERNEL); if (ret) { diff --git a/drivers/remoteproc/qcom_q6v5_mss.c b/drivers/remoteproc/qcom_q6v5_mss.c index bef198b9ee63..2f71ed2feff6 100644 --- a/drivers/remoteproc/qcom_q6v5_mss.c +++ b/drivers/remoteproc/qcom_q6v5_mss.c @@ -2079,7 +2079,16 @@ static int q6v5_probe(struct platform_device *pdev) if (!desc) return -EINVAL; - if (desc->need_mem_protection && !qcom_pas_is_available()) + /* + * Memory protection is done through qcom_scm_assign_mem(), which needs + * SCM but not PAS. Only the memory setup path issues PAS calls, so + * requiring PAS for every need_mem_protection platform prevents the + * modem from probing at all on TZ firmware that offers no PAS. + */ + if (desc->need_mem_protection && !qcom_scm_is_available()) + return -EPROBE_DEFER; + + if (desc->need_pas_mem_setup && !qcom_pas_is_available()) return -EPROBE_DEFER; mba_image = desc->hexagon_mba_image; diff --git a/drivers/remoteproc/qcom_q6v5_pas.c b/drivers/remoteproc/qcom_q6v5_pas.c index ca8e61254c44..a005546c265d 100644 --- a/drivers/remoteproc/qcom_q6v5_pas.c +++ b/drivers/remoteproc/qcom_q6v5_pas.c @@ -405,6 +405,7 @@ static int qcom_pas_stop(struct rproc *rproc) { struct qcom_pas *pas = rproc->priv; int handover; + int dtb_ret; int ret; ret = qcom_q6v5_request_stop(&pas->q6v5, pas->sysmon); @@ -419,9 +420,12 @@ static int qcom_pas_stop(struct rproc *rproc) dev_err(pas->dev, "failed to shutdown: %d\n", ret); if (pas->dtb_pas_id) { - ret = qcom_pas_shutdown(pas->dtb_pas_id); - if (ret) - dev_err(pas->dev, "failed to shutdown dtb: %d\n", ret); + dtb_ret = qcom_pas_shutdown(pas->dtb_pas_id); + if (dtb_ret) + dev_err(pas->dev, "failed to shutdown dtb: %d\n", dtb_ret); + + if (!ret && dtb_ret) + ret = dtb_ret; qcom_pas_unmap_carveout(rproc, pas->dtb_mem_phys, pas->dtb_mem_size); } @@ -524,7 +528,6 @@ static int qcom_pas_attach(struct rproc *rproc) int ret; pas->q6v5.handover_issued = true; - enable_irq(pas->q6v5.handover_irq); pas->q6v5.running = true; ret = irq_get_irqchip_state(pas->q6v5.fatal_irq, @@ -570,7 +573,6 @@ static int qcom_pas_attach(struct rproc *rproc) pas->rproc->state = RPROC_OFFLINE; ret = -EINVAL; disable_running: - disable_irq(pas->q6v5.handover_irq); pas->q6v5.running = false; return ret; diff --git a/drivers/rtc/dev.c b/drivers/rtc/dev.c index 8ba7c25d2565..be173c9511fa 100644 --- a/drivers/rtc/dev.c +++ b/drivers/rtc/dev.c @@ -216,7 +216,7 @@ static long rtc_dev_ioctl(struct file *file, struct rtc_device *rtc = file->private_data; const struct rtc_class_ops *ops = rtc->ops; struct rtc_time tm; - struct rtc_wkalrm alarm; + struct rtc_wkalrm alarm = { 0 }; struct rtc_param param; void __user *uarg = (void __user *)arg; diff --git a/drivers/rtc/rtc-ac100.c b/drivers/rtc/rtc-ac100.c index bba7115ff3ad..c579c4bf5578 100644 --- a/drivers/rtc/rtc-ac100.c +++ b/drivers/rtc/rtc-ac100.c @@ -317,10 +317,12 @@ static int ac100_rtc_register_clks(struct ac100_rtc_dev *chip) if (!chip->clk_data) return -ENOMEM; - chip->rtc_32k_clk = clk_hw_register_fixed_rate(chip->dev, - AC100_RTC_32K_NAME, - NULL, 0, - AC100_RTC_32K_RATE); + chip->clk_data->num = AC100_CLKOUT_NUM; + + chip->rtc_32k_clk = devm_clk_hw_register_fixed_rate(chip->dev, + AC100_RTC_32K_NAME, + NULL, 0, + AC100_RTC_32K_RATE); if (IS_ERR(chip->rtc_32k_clk)) { ret = PTR_ERR(chip->rtc_32k_clk); dev_err(chip->dev, "Failed to register RTC-32k clock: %d\n", @@ -354,29 +356,14 @@ static int ac100_rtc_register_clks(struct ac100_rtc_dev *chip) if (ret) { dev_err(chip->dev, "Failed to register clk '%s': %d\n", init.name, ret); - goto err_unregister_rtc_32k; + return ret; } chip->clk_data->hws[i] = &clk->hw; } - chip->clk_data->num = i; - ret = of_clk_add_hw_provider(np, of_clk_hw_onecell_get, chip->clk_data); - if (ret) - goto err_unregister_rtc_32k; - - return 0; - -err_unregister_rtc_32k: - clk_unregister_fixed_rate(chip->rtc_32k_clk->clk); - - return ret; -} - -static void ac100_rtc_unregister_clks(struct ac100_rtc_dev *chip) -{ - of_clk_del_provider(chip->dev->of_node); - clk_unregister_fixed_rate(chip->rtc_32k_clk->clk); + return devm_of_clk_add_hw_provider(chip->dev, of_clk_hw_onecell_get, + chip->clk_data); } /* @@ -614,13 +601,6 @@ static int ac100_rtc_probe(struct platform_device *pdev) return devm_rtc_register_device(chip->rtc); } -static void ac100_rtc_remove(struct platform_device *pdev) -{ - struct ac100_rtc_dev *chip = platform_get_drvdata(pdev); - - ac100_rtc_unregister_clks(chip); -} - static const struct of_device_id ac100_rtc_match[] = { { .compatible = "x-powers,ac100-rtc" }, { }, @@ -629,7 +609,6 @@ MODULE_DEVICE_TABLE(of, ac100_rtc_match); static struct platform_driver ac100_rtc_driver = { .probe = ac100_rtc_probe, - .remove = ac100_rtc_remove, .driver = { .name = "ac100-rtc", .of_match_table = of_match_ptr(ac100_rtc_match), diff --git a/drivers/rtc/rtc-efi.c b/drivers/rtc/rtc-efi.c index b4f44999ef0f..a8c18ebc5599 100644 --- a/drivers/rtc/rtc-efi.c +++ b/drivers/rtc/rtc-efi.c @@ -52,6 +52,7 @@ compute_wday(efi_time_t *eft, int yday) static void convert_to_efi_time(struct rtc_time *wtime, efi_time_t *eft) { + memset(eft, 0, sizeof(*eft)); eft->year = wtime->tm_year + 1900; eft->month = wtime->tm_mon + 1; eft->day = wtime->tm_mday; @@ -112,6 +113,35 @@ convert_from_efi_time(efi_time_t *eft, struct rtc_time *wtime) return true; } +static int efi_read_alarm(struct device *dev, struct rtc_wkalrm *wkalrm) +{ + efi_time_t eft; + efi_status_t status; + + memset(&eft, 0, sizeof(eft)); + status = efi.get_wakeup_time((efi_bool_t *)&wkalrm->enabled, + (efi_bool_t *)&wkalrm->pending, &eft); + if (status != EFI_SUCCESS) + return -EINVAL; + + if (!convert_from_efi_time(&eft, &wkalrm->time)) + return -EIO; + + return rtc_valid_tm(&wkalrm->time); +} + +static int efi_set_alarm(struct device *dev, struct rtc_wkalrm *wkalrm) +{ + efi_time_t eft; + efi_status_t status; + + convert_to_efi_time(&wkalrm->time, &eft); + + status = efi.set_wakeup_time((efi_bool_t)!!wkalrm->enabled, &eft); + + return status == EFI_SUCCESS ? 0 : -EINVAL; +} + static int efi_read_time(struct device *dev, struct rtc_time *tm) { efi_status_t status; @@ -146,13 +176,21 @@ static int efi_set_time(struct device *dev, struct rtc_time *tm) static int efi_procfs(struct device *dev, struct seq_file *seq) { - efi_time_t eft; + efi_time_t eft, alm; efi_time_cap_t cap; + efi_bool_t enabled, pending; + struct rtc_device *rtc = dev_get_drvdata(dev); memset(&eft, 0, sizeof(eft)); + memset(&alm, 0, sizeof(alm)); memset(&cap, 0, sizeof(cap)); efi.get_time(&eft, &cap); + if (test_bit(RTC_FEATURE_ALARM, rtc->features) && + efi.get_wakeup_time(&enabled, &pending, &alm) != EFI_SUCCESS) { + enabled = 0; + pending = 0; + } seq_printf(seq, "Time\t\t: %u:%u:%u.%09u\n" @@ -168,6 +206,25 @@ static int efi_procfs(struct device *dev, struct seq_file *seq) /* XXX fixme: convert to string? */ seq_printf(seq, "Timezone\t: %u\n", eft.timezone); + if (test_bit(RTC_FEATURE_ALARM, rtc->features)) { + seq_printf(seq, + "Alarm Time\t: %u:%u:%u.%09u\n" + "Alarm Date\t: %u-%u-%u\n" + "Alarm Daylight\t: %u\n" + "Enabled\t\t: %s\n" + "Pending\t\t: %s\n", + alm.hour, alm.minute, alm.second, alm.nanosecond, + alm.year, alm.month, alm.day, + alm.daylight, + enabled == 1 ? "yes" : "no", + pending == 1 ? "yes" : "no"); + + if (alm.timezone == EFI_UNSPECIFIED_TIMEZONE) + seq_puts(seq, "Alarm Timezone\t: unspecified\n"); + else + seq_printf(seq, "Alarm Timezone\t: %d\n", alm.timezone); + } + /* * now prints the capabilities */ @@ -183,6 +240,8 @@ static int efi_procfs(struct device *dev, struct seq_file *seq) static const struct rtc_class_ops efi_rtc_ops = { .read_time = efi_read_time, .set_time = efi_set_time, + .read_alarm = efi_read_alarm, + .set_alarm = efi_set_alarm, .proc = efi_procfs, }; @@ -191,6 +250,7 @@ static int __init efi_rtc_probe(struct platform_device *dev) struct rtc_device *rtc; efi_time_t eft; efi_time_cap_t cap; + efi_bool_t enabled, pending; /* First check if the RTC is usable */ if (efi.get_time(&eft, &cap) != EFI_SUCCESS) @@ -203,7 +263,23 @@ static int __init efi_rtc_probe(struct platform_device *dev) platform_set_drvdata(dev, rtc); rtc->ops = &efi_rtc_ops; - clear_bit(RTC_FEATURE_ALARM, rtc->features); + clear_bit(RTC_FEATURE_UPDATE_INTERRUPT, rtc->features); + + /* + * The EFI_RT_SUPPORTED_WAKEUP_SERVICES bit defaults to enabled + * and only gets cleared when the RT_PROP table explicitly says + * wakeup is unsupported. Many platforms lack an RT_PROP table + * even though they don't implement the wakeup runtime service, + * so probe by actually calling GetWakeupTime() to avoid exposing + * a broken alarm to userspace. + */ + if (efi_rt_services_supported(EFI_RT_SUPPORTED_WAKEUP_SERVICES) && + efi.get_wakeup_time(&enabled, &pending, &eft) == EFI_SUCCESS) { + set_bit(RTC_FEATURE_ALARM, rtc->features); + set_bit(RTC_FEATURE_ALARM_WAKEUP_ONLY, rtc->features); + } else { + clear_bit(RTC_FEATURE_ALARM, rtc->features); + } device_init_wakeup(&dev->dev, true); diff --git a/drivers/rtc/rtc-mpfs.c b/drivers/rtc/rtc-mpfs.c index ece6de4a6adb..60596b0ea9d8 100644 --- a/drivers/rtc/rtc-mpfs.c +++ b/drivers/rtc/rtc-mpfs.c @@ -256,8 +256,12 @@ static int mpfs_rtc_probe(struct platform_device *pdev) return ret; } + clk = devm_clk_get(&pdev->dev, "rtcref"); + if (IS_ERR(clk)) + return PTR_ERR(clk); + /* prescaler hardware adds 1 to reg value */ - prescaler = clk_get_rate(devm_clk_get(&pdev->dev, "rtcref")) - 1; + prescaler = clk_get_rate(clk) - 1; if (prescaler > MAX_PRESCALER_COUNT) { dev_dbg(&pdev->dev, "invalid prescaler %lu\n", prescaler); return -EINVAL; diff --git a/drivers/rtc/rtc-spear.c b/drivers/rtc/rtc-spear.c index 959acff8faff..d12bccb3cf5e 100644 --- a/drivers/rtc/rtc-spear.c +++ b/drivers/rtc/rtc-spear.c @@ -363,6 +363,12 @@ static int spear_rtc_probe(struct platform_device *pdev) if (irq < 0) return irq; + config->ioaddr = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(config->ioaddr)) + return PTR_ERR(config->ioaddr); + + spin_lock_init(&config->lock); + status = devm_request_irq(&pdev->dev, irq, spear_rtc_irq, 0, pdev->name, config); if (status) { @@ -371,10 +377,6 @@ static int spear_rtc_probe(struct platform_device *pdev) return status; } - config->ioaddr = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(config->ioaddr)) - return PTR_ERR(config->ioaddr); - config->clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(config->clk)) return PTR_ERR(config->clk); @@ -383,7 +385,6 @@ static int spear_rtc_probe(struct platform_device *pdev) if (status < 0) return status; - spin_lock_init(&config->lock); platform_set_drvdata(pdev, config); config->rtc->ops = &spear_rtc_ops; diff --git a/drivers/s390/block/dasd_3990_erp.c b/drivers/s390/block/dasd_3990_erp.c index 736459477c19..121cb9ae5c67 100644 --- a/drivers/s390/block/dasd_3990_erp.c +++ b/drivers/s390/block/dasd_3990_erp.c @@ -103,7 +103,7 @@ dasd_3990_erp_int_req(struct dasd_ccw_req * erp) /* first time set initial retry counter and erp_function */ /* and retry once without blocking queue */ - /* (this enables easier enqueing of the cqr) */ + /* (this enables easier enqueuing of the cqr) */ if (erp->function != dasd_3990_erp_int_req) { erp->retries = 256; @@ -302,7 +302,7 @@ dasd_3990_erp_action_4(struct dasd_ccw_req * erp, char *sense) /* first time set initial retry counter and erp_function */ /* and retry once without waiting for state change pending */ - /* interrupt (this enables easier enqueing of the cqr) */ + /* interrupt (this enables easier enqueuing of the cqr) */ if (erp->function != dasd_3990_erp_action_4) { DBF_DEV_EVENT(DBF_INFO, device, "%s", @@ -1078,7 +1078,7 @@ dasd_3990_erp_bus_out(struct dasd_ccw_req * erp) /* first time set initial retry counter and erp_function */ /* and retry once without blocking queue */ - /* (this enables easier enqueing of the cqr) */ + /* (this enables easier enqueuing of the cqr) */ if (erp->function != dasd_3990_erp_bus_out) { erp->retries = 256; erp->function = dasd_3990_erp_bus_out; diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index 8d976dd58a6c..9d1b11020b4e 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -2060,7 +2060,7 @@ dasd_eckd_psf_ssc(struct dasd_device *device, int enable_pav, } /* - * Valide storage server of current device. + * Valid storage server of current device. */ static int dasd_eckd_validate_server(struct dasd_device *device, unsigned long flags) @@ -5672,7 +5672,7 @@ static struct dasd_ccw_req *dasd_eckd_build_cp_raw(struct dasd_device *startdev, char *dst; /* - * raw track access needs to be mutiple of 64k and on 64k boundary + * raw track access needs to be multiple of 64k and on 64k boundary * For read requests we can fix an incorrect alignment by padding * the request with dummy pages. */ diff --git a/drivers/s390/char/raw3270.c b/drivers/s390/char/raw3270.c index aa9c4d81225c..6861bf27d15f 100644 --- a/drivers/s390/char/raw3270.c +++ b/drivers/s390/char/raw3270.c @@ -420,7 +420,7 @@ struct raw3270_ua { /* Query Reply structure for Usable Area */ char flags0; char flags1; short w; /* Width of usable area */ - short h; /* Heigth of usavle area */ + short h; /* Height of usavle area */ char units; /* 0x00:in; 0x01:mm */ int xr; int yr; diff --git a/drivers/s390/char/vmlogrdr.c b/drivers/s390/char/vmlogrdr.c index 383e7e2bd69f..005735a89920 100644 --- a/drivers/s390/char/vmlogrdr.c +++ b/drivers/s390/char/vmlogrdr.c @@ -444,7 +444,7 @@ static int vmlogrdr_receive_data(struct vmlogrdr_priv_t *priv) spin_unlock_bh(&priv->priv_lock); /* An rc of 5 indicates that the record was bigger than * the buffer, which is OK for us. A 9 indicates that the - * record was purged befor we could receive it. + * record was purged before we could receive it. */ if (rc == 5) rc = 0; diff --git a/drivers/s390/cio/chp.c b/drivers/s390/cio/chp.c index c890f21a82ce..eaf0527bff6c 100644 --- a/drivers/s390/cio/chp.c +++ b/drivers/s390/cio/chp.c @@ -78,6 +78,9 @@ u8 chp_get_sch_opm(struct subchannel *sch) int opm; int i; + if (!sch->schib.pmcw.dnv) + return 0; + opm = 0; chp_id_init(&chpid); for (i = 0; i < 8; i++) { diff --git a/drivers/s390/cio/cio.c b/drivers/s390/cio/cio.c index 70dc8cc76594..e1c62eb60cca 100644 --- a/drivers/s390/cio/cio.c +++ b/drivers/s390/cio/cio.c @@ -453,7 +453,8 @@ EXPORT_SYMBOL_GPL(cio_commit_config); /** * cio_update_schib - Perform stsch and update schib if subchannel is valid. * @sch: subchannel on which to perform stsch - * Return zero on success, -ENODEV otherwise. + * Return zero on success, -ENODEV if the subchannel is not operational, + * -EACCES if the subchannel has no valid device. */ int cio_update_schib(struct subchannel *sch) { @@ -462,10 +463,12 @@ int cio_update_schib(struct subchannel *sch) if (stsch(sch->schid, &schib)) return -ENODEV; - memcpy(&sch->schib, &schib, sizeof(schib)); - - if (!css_sch_is_valid(&schib)) + if (!css_sch_is_valid(&schib)) { + memset(&sch->schib, 0, sizeof(sch->schib)); return -EACCES; + } + + memcpy(&sch->schib, &schib, sizeof(schib)); return 0; } diff --git a/drivers/s390/cio/cio.h b/drivers/s390/cio/cio.h index bad142c536e1..6d28a62bc67a 100644 --- a/drivers/s390/cio/cio.h +++ b/drivers/s390/cio/cio.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -49,7 +50,7 @@ struct pmcw { /* Target SCHIB configuration. */ struct schib_config { - u64 mba; + dma64_t mba; u32 intparm; u16 mbi; u32 isc:3; @@ -66,7 +67,7 @@ struct schib_config { struct schib { struct pmcw pmcw; /* path management control word */ union scsw scsw; /* subchannel status word */ - __u64 mba; /* measurement block address */ + dma64_t mba; /* measurement block address */ __u8 mda[4]; /* model dependent area */ } __attribute__ ((packed,aligned(4))); diff --git a/drivers/s390/cio/cmf.c b/drivers/s390/cio/cmf.c index 92ab3d546fe4..66b14fedbd18 100644 --- a/drivers/s390/cio/cmf.c +++ b/drivers/s390/cio/cmf.c @@ -183,7 +183,7 @@ static int set_schib(struct ccw_device *cdev, u32 mme, int mbfc, sch->config.mbfc = mbfc; /* address can be either a block address or a block index */ if (mbfc) - sch->config.mba = address; + sch->config.mba = address ? virt_to_dma64((void *)address) : 0; else sch->config.mbi = address; diff --git a/drivers/s390/cio/device.c b/drivers/s390/cio/device.c index fb591118ecb2..68dd4a62975d 100644 --- a/drivers/s390/cio/device.c +++ b/drivers/s390/cio/device.c @@ -922,7 +922,7 @@ static int ccw_device_move_to_sch(struct ccw_device *cdev, if (!sch_is_pseudo_sch(old_sch)) { spin_lock_irq(&old_sch->lock); - old_enabled = old_sch->schib.pmcw.ena; + old_enabled = old_sch->schib.pmcw.dnv && old_sch->schib.pmcw.ena; rc = 0; if (old_enabled) rc = cio_disable_subchannel(old_sch); @@ -941,7 +941,7 @@ static int ccw_device_move_to_sch(struct ccw_device *cdev, CIO_MSG_EVENT(0, "device_move(0.%x.%04x,0.%x.%04x)=%d\n", cdev->private->dev_id.ssid, cdev->private->dev_id.devno, sch->schid.ssid, - sch->schib.pmcw.dev, rc); + sch->schid.sch_no, rc); if (old_enabled) { /* Try to re-enable the old subchannel. */ spin_lock_irq(&old_sch->lock); @@ -1207,7 +1207,7 @@ static void io_subchannel_quiesce(struct subchannel *sch) cdev = sch_get_cdev(sch); if (cio_is_console(sch->schid)) goto out_unlock; - if (!sch->schib.pmcw.ena) + if (!sch->schib.pmcw.dnv || !sch->schib.pmcw.ena) goto out_unlock; ret = cio_disable_subchannel(sch); if (ret != -EBUSY) @@ -1254,7 +1254,8 @@ static int recovery_check(struct device *dev, void *data) switch (cdev->private->state) { case DEV_STATE_ONLINE: sch = to_subchannel(cdev->dev.parent); - if ((sch->schib.pmcw.pam & sch->opm) == sch->vpm) + if (sch->schib.pmcw.dnv && + (sch->schib.pmcw.pam & sch->opm) == sch->vpm) break; fallthrough; case DEV_STATE_DISCONNECTED: diff --git a/drivers/s390/cio/device_fsm.c b/drivers/s390/cio/device_fsm.c index ab419d40a8a7..b5686c25c83c 100644 --- a/drivers/s390/cio/device_fsm.c +++ b/drivers/s390/cio/device_fsm.c @@ -170,6 +170,9 @@ __recover_lost_chpids(struct subchannel *sch, int old_lpm) int mask, i; struct chp_id chpid; + if (!sch->schib.pmcw.dnv) + return; + chp_id_init(&chpid); for (i = 0; i<8; i++) { mask = 0x80 >> i; diff --git a/drivers/s390/cio/device_ops.c b/drivers/s390/cio/device_ops.c index 61c07b4a0fe8..1f7e83fd5087 100644 --- a/drivers/s390/cio/device_ops.c +++ b/drivers/s390/cio/device_ops.c @@ -142,6 +142,8 @@ int ccw_device_clear(struct ccw_device *cdev, unsigned long intparm) if (!cdev || !cdev->dev.parent) return -ENODEV; sch = to_subchannel(cdev->dev.parent); + if (!sch->schib.pmcw.dnv) + return -ENODEV; if (!sch->schib.pmcw.ena) return -EINVAL; if (cdev->private->state == DEV_STATE_NOT_OPER) @@ -198,6 +200,8 @@ int ccw_device_start_timeout_key(struct ccw_device *cdev, struct ccw1 *cpa, if (!cdev || !cdev->dev.parent) return -ENODEV; sch = to_subchannel(cdev->dev.parent); + if (!sch->schib.pmcw.dnv) + return -ENODEV; if (!sch->schib.pmcw.ena) return -EINVAL; if (cdev->private->state == DEV_STATE_NOT_OPER) @@ -379,6 +383,8 @@ int ccw_device_halt(struct ccw_device *cdev, unsigned long intparm) if (!cdev || !cdev->dev.parent) return -ENODEV; sch = to_subchannel(cdev->dev.parent); + if (!sch->schib.pmcw.dnv) + return -ENODEV; if (!sch->schib.pmcw.ena) return -EINVAL; if (cdev->private->state == DEV_STATE_NOT_OPER) @@ -413,6 +419,8 @@ int ccw_device_resume(struct ccw_device *cdev) if (!cdev || !cdev->dev.parent) return -ENODEV; sch = to_subchannel(cdev->dev.parent); + if (!sch->schib.pmcw.dnv) + return -ENODEV; if (!sch->schib.pmcw.ena) return -EINVAL; if (cdev->private->state == DEV_STATE_NOT_OPER) @@ -482,6 +490,8 @@ struct channel_path_desc_fmt0 *ccw_device_get_chp_desc(struct ccw_device *cdev, struct chp_id chpid; sch = to_subchannel(cdev->dev.parent); + if (!sch->schib.pmcw.dnv) + return NULL; chp_id_init(&chpid); chpid.id = sch->schib.pmcw.chpid[chp_idx]; return chp_get_chp_desc(chpid); @@ -502,9 +512,13 @@ u8 *ccw_device_get_util_str(struct ccw_device *cdev, int chp_idx) struct chp_id chpid; u8 *util_str; + if (!sch->schib.pmcw.dnv) + return NULL; chp_id_init(&chpid); chpid.id = sch->schib.pmcw.chpid[chp_idx]; chp = chpid_to_chp(chpid); + if (!chp) + return NULL; util_str = kmalloc(sizeof(chp->desc_fmt3.util_str), GFP_KERNEL); if (!util_str) @@ -548,6 +562,8 @@ int ccw_device_tm_start_timeout_key(struct ccw_device *cdev, struct tcw *tcw, int rc; sch = to_subchannel(cdev->dev.parent); + if (!sch->schib.pmcw.dnv) + return -ENODEV; if (!sch->schib.pmcw.ena) return -EINVAL; if (cdev->private->state == DEV_STATE_VERIFY) { @@ -652,6 +668,9 @@ int ccw_device_get_mdc(struct ccw_device *cdev, u8 mask) struct chp_id chpid; int mdc = 0, i; + if (!sch->schib.pmcw.dnv) + return 0; + /* Adjust requested path mask to excluded varied off paths. */ if (mask) mask &= sch->lpm; @@ -694,6 +713,8 @@ int ccw_device_tm_intrg(struct ccw_device *cdev) { struct subchannel *sch = to_subchannel(cdev->dev.parent); + if (!sch->schib.pmcw.dnv) + return -ENODEV; if (!sch->schib.pmcw.ena) return -EINVAL; if (cdev->private->state != DEV_STATE_ONLINE) @@ -786,6 +807,8 @@ int ccw_device_get_chpid(struct ccw_device *cdev, int chp_idx, u8 *chpid) if ((chp_idx < 0) || (chp_idx > 7)) return -EINVAL; + if (!sch->schib.pmcw.dnv) + return -ENODEV; mask = 0x80 >> chp_idx; if (!(sch->schib.pmcw.pim & mask)) return -ENODEV; diff --git a/drivers/s390/cio/vfio_ccw_fsm.c b/drivers/s390/cio/vfio_ccw_fsm.c index 5fd94e9d5c61..9a000b0231d6 100644 --- a/drivers/s390/cio/vfio_ccw_fsm.c +++ b/drivers/s390/cio/vfio_ccw_fsm.c @@ -399,7 +399,7 @@ static void fsm_close(struct vfio_ccw_private *private, spin_lock_irq(&sch->lock); - if (!sch->schib.pmcw.ena) + if (!sch->schib.pmcw.dnv || !sch->schib.pmcw.ena) goto err_unlock; ret = cio_disable_subchannel(sch); diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 940c0ff668be..4db878c18f41 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -277,9 +277,9 @@ static void vfio_ap_free_aqic_resources(struct vfio_ap_queue *q) { if (!q) return; - if (q->saved_isc != VFIO_AP_ISC_INVALID && - !WARN_ON(!(q->matrix_mdev && q->matrix_mdev->kvm))) { - kvm_s390_gisc_unregister(q->matrix_mdev->kvm, q->saved_isc); + if (q->saved_isc != VFIO_AP_ISC_INVALID) { + if (!WARN_ON(!q->matrix_mdev) && q->matrix_mdev->kvm) + kvm_s390_gisc_unregister(q->matrix_mdev->kvm, q->saved_isc); q->saved_isc = VFIO_AP_ISC_INVALID; } if (q->saved_iova && !WARN_ON(!q->matrix_mdev)) { @@ -1935,6 +1935,8 @@ static int apq_status_check(int apqn, struct ap_queue_status *status) * a value indicating a reset needs to be performed again. */ return -EAGAIN; + case AP_RESPONSE_Q_NOT_AVAIL: + return -ENODEV; default: WARN(true, "failed to verify reset of queue %02x.%04x: TAPQ rc=%u\n", @@ -1961,6 +1963,10 @@ static void apq_reset_check(struct work_struct *reset_work) ret = apq_status_check(q->apqn, &status); if (ret == -EIO) return; + if (ret == -ENODEV) { + vfio_ap_free_aqic_resources(q); + return; + } if (ret == -EBUSY) { pr_notice_ratelimited(WAIT_MSG, elapsed, AP_QID_CARD(q->apqn), @@ -2004,6 +2010,7 @@ static void vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q) break; case AP_RESPONSE_DECONFIGURED: case AP_RESPONSE_CHECKSTOPPED: + case AP_RESPONSE_Q_NOT_AVAIL: vfio_ap_free_aqic_resources(q); break; default: @@ -2528,12 +2535,15 @@ void vfio_ap_mdev_remove_queue(struct ap_device *apdev) /* * If the queue is not in the host's AP configuration, then resetting * it will fail with response code 01, (APQN not valid); so, let's make - * sure it is in the host's config. + * sure it is in the host's config. If it is not, free the KVM GISC + * resources. */ if (test_bit_inv(apid, (unsigned long *)matrix_dev->info.apm) && test_bit_inv(apqi, (unsigned long *)matrix_dev->info.aqm)) { vfio_ap_mdev_reset_queue(q); flush_work(&q->reset_work); + } else { + vfio_ap_free_aqic_resources(q); } done: diff --git a/drivers/scsi/leapraid/leapraid_func.h b/drivers/scsi/leapraid/leapraid_func.h index 4c0b9ca728d8..e8a0815bf95f 100644 --- a/drivers/scsi/leapraid/leapraid_func.h +++ b/drivers/scsi/leapraid/leapraid_func.h @@ -554,7 +554,6 @@ struct leapraid_fw_evt_work { /** * struct leapraid_fw_evt_struct - Firmware event handling structure * - * @fw_evt_name: Name of the firmware event. * @fw_evt_thread: Workqueue used for processing firmware events. * @fw_evt_lock: Spinlock protecting access to the firmware event list. * @fw_evt_list: Linked list of pending firmware events. @@ -565,7 +564,6 @@ struct leapraid_fw_evt_work { */ struct leapraid_fw_evt_struct { u32 leapraid_evt_masks[4]; - char fw_evt_name[48]; struct workqueue_struct *fw_evt_thread; spinlock_t fw_evt_lock; /* protects firmware event */ struct list_head fw_evt_list; diff --git a/drivers/scsi/leapraid/leapraid_os.c b/drivers/scsi/leapraid/leapraid_os.c index ee3242779dfd..507f11862276 100644 --- a/drivers/scsi/leapraid/leapraid_os.c +++ b/drivers/scsi/leapraid/leapraid_os.c @@ -2054,12 +2054,10 @@ static int leapraid_probe(struct pci_dev *pdev, const struct pci_device_id *id) shost->transportt = leapraid_transport_template; shost->unique_id = adapter->adapter_attr.id; - snprintf(adapter->fw_evt_s.fw_evt_name, - sizeof(adapter->fw_evt_s.fw_evt_name), - "fw_event_%s%d", LEAPRAID_DRIVER_NAME, - adapter->adapter_attr.id); adapter->fw_evt_s.fw_evt_thread = - alloc_ordered_workqueue(adapter->fw_evt_s.fw_evt_name, 0); + alloc_ordered_workqueue("fw_event_%s%d", 0, + LEAPRAID_DRIVER_NAME, + adapter->adapter_attr.id); if (!adapter->fw_evt_s.fw_evt_thread) { dev_err(&adapter->pdev->dev, "%s: Failed to create fw event workqueue\n", __func__); diff --git a/drivers/scsi/libiscsi_tcp.c b/drivers/scsi/libiscsi_tcp.c index 7223bb18b048..d35f93451ee9 100644 --- a/drivers/scsi/libiscsi_tcp.c +++ b/drivers/scsi/libiscsi_tcp.c @@ -480,6 +480,9 @@ static int iscsi_tcp_data_in(struct iscsi_conn *conn, struct iscsi_task *task) int datasn = be32_to_cpu(rhdr->datasn); unsigned total_in_length = task->sc->sdb.length; + if (task->sc->sc_data_direction != DMA_FROM_DEVICE) + return ISCSI_ERR_PROTO; + /* * lib iscsi will update this in the completion handling if there * is status. diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c index b95f187297ae..4f9a53769966 100644 --- a/drivers/scsi/megaraid/megaraid_sas_base.c +++ b/drivers/scsi/megaraid/megaraid_sas_base.c @@ -7886,7 +7886,9 @@ megasas_resume(struct device *dev) goto fail_init_mfi; } - if (megasas_get_ctrl_info(instance) != DCMD_SUCCESS) + scoped_guard(mutex, &instance->reset_mutex) + rval = megasas_get_ctrl_info(instance); + if (rval != DCMD_SUCCESS) goto fail_init_mfi; tasklet_init(&instance->isr_tasklet, instance->instancet->tasklet, diff --git a/drivers/scsi/scsi_devinfo.c b/drivers/scsi/scsi_devinfo.c index 15ffbe93ac72..88a911b35c94 100644 --- a/drivers/scsi/scsi_devinfo.c +++ b/drivers/scsi/scsi_devinfo.c @@ -161,7 +161,7 @@ static struct { {"DGC", "DISK", NULL, BLIST_SPARSELUN}, /* EMC CLARiiON, no storage on LUN 0 */ {"EMC", "Invista", "*", BLIST_SPARSELUN | BLIST_LARGELUN}, {"EMC", "SYMMETRIX", NULL, BLIST_SPARSELUN | BLIST_LARGELUN | - BLIST_REPORTLUN2 | BLIST_RETRY_ITF}, + BLIST_REPORTLUN2 | BLIST_RETRY_ITF | BLIST_SKIP_IO_HINTS}, {"EMULEX", "MD21/S2 ESDI", NULL, BLIST_SINGLELUN}, {"easyRAID", "16P", NULL, BLIST_NOREPORTLUN}, {"easyRAID", "X6P", NULL, BLIST_NOREPORTLUN}, diff --git a/drivers/scsi/sd_zbc.c b/drivers/scsi/sd_zbc.c index 56e455fb5add..456beaf2e769 100644 --- a/drivers/scsi/sd_zbc.c +++ b/drivers/scsi/sd_zbc.c @@ -589,7 +589,7 @@ int sd_zbc_revalidate_zones(struct scsi_disk *sdkp) int sd_zbc_read_zones(struct scsi_disk *sdkp, struct queue_limits *lim, u8 buf[SD_BUF_SIZE]) { - unsigned int nr_zones; + u64 nr_zones; u32 zone_blocks = 0; int ret; @@ -621,6 +621,12 @@ int sd_zbc_read_zones(struct scsi_disk *sdkp, struct queue_limits *lim, goto err; nr_zones = round_up(sdkp->capacity, zone_blocks) >> ilog2(zone_blocks); + if (nr_zones > INT_MAX) { + sd_printk(KERN_ERR, sdkp, "Too many zones (%llu)\n", + nr_zones); + ret = -EINVAL; + goto err; + } sdkp->early_zone_info.nr_zones = nr_zones; sdkp->early_zone_info.zone_blocks = zone_blocks; diff --git a/drivers/spi/spi-cs42l43.c b/drivers/spi/spi-cs42l43.c index 7106a8a4f280..6772abb483dd 100644 --- a/drivers/spi/spi-cs42l43.c +++ b/drivers/spi/spi-cs42l43.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -215,12 +216,108 @@ static size_t cs42l43_spi_max_length(struct spi_device *spi) return CS42L43_SPI_MAX_LENGTH; } +/* + * Workaround needed for two speaker ID pins in one ACPI GpioIo() but + * the Linux-specific _DSD property only contains one pin. + * Create a temporary acpi_gpio_mapping pointing at both pins. + */ +static const struct acpi_gpio_params cs42l43_2bit_speaker_id_from_one_gpioio_params[] = { + [0] = { + .crs_entry_index = 0, + .line_index = 0, + }, + [1] = { + .crs_entry_index = 0, + .line_index = 1, + }, +}; + +static const struct acpi_gpio_mapping cs42l43_2bit_speaker_id_from_one_gpioio_mapping[] = { + { + .name = "spk-id-quirk-gpios", + .data = cs42l43_2bit_speaker_id_from_one_gpioio_params, + .size = ARRAY_SIZE(cs42l43_2bit_speaker_id_from_one_gpioio_params), + }, + { } +}; + +static int cs42l43_get_2bit_speaker_id_from_one_gpioio(struct cs42l43_spi *priv, int *result) +{ + struct fwnode_reference_args args; + struct acpi_device *adev; + struct gpio_desc *desc; + u32 spkid = 0; + int i, ret; + + /* Use the _DSD property to get the node containing the GpioIo() */ + ret = fwnode_property_get_reference_args(dev_fwnode(priv->dev), "spk-id-gpios", + NULL, 3, 0, &args); + if (ret) + return ret; + + struct fwnode_handle *fwnode __free(fwnode_handle) = args.fwnode; + + /* An acpi_gpio_mapping must be added to the node that contains the GpioIo() */ + adev = to_acpi_device_node(fwnode); + if (!adev) + return -EINVAL; + + ret = acpi_dev_add_driver_gpios(adev, cs42l43_2bit_speaker_id_from_one_gpioio_mapping); + if (ret) + return ret; + + /* gpiod_get_array() can't read from a mapping in a child node */ + for (i = 0; i < ARRAY_SIZE(cs42l43_2bit_speaker_id_from_one_gpioio_params); i++) { + desc = fwnode_gpiod_get_index(fwnode, "spk-id-quirk", i, GPIOD_IN, + dev_name(priv->dev)); + if (IS_ERR(desc)) { + ret = PTR_ERR(desc); + goto out; + } + + ret = gpiod_get_value_cansleep(desc); + gpiod_put(desc); + if (ret < 0) + goto out; + + spkid |= (u32)ret << i; + } + + dev_dbg(priv->dev, "spk-id-gpios = %u\n", spkid); + *result = spkid; + ret = 0; +out: + acpi_dev_remove_driver_gpios(adev); + + return ret; +} + +static const struct dmi_system_id cs42l43_spk_id_quirks[] = { + { + .ident = "Dell XPS 13 DX13260", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc"), + DMI_EXACT_MATCH(DMI_PRODUCT_SKU, "0E53"), + }, + .driver_data = cs42l43_get_2bit_speaker_id_from_one_gpioio, + }, + { } +}; + static int cs42l43_get_speaker_id_gpios(struct cs42l43_spi *priv, int *result) { + const struct dmi_system_id *dmi_id; struct gpio_descs *descs; u32 spkid; int i, ret; + dmi_id = dmi_first_match(cs42l43_spk_id_quirks); + if (dmi_id) { + int (*get_speaker_id)(struct cs42l43_spi *priv, int *result) = dmi_id->driver_data; + + return get_speaker_id(priv, result); + } + descs = gpiod_get_array_optional(priv->dev, "spk-id", GPIOD_IN); if (!descs) return 0; diff --git a/drivers/staging/greybus/uart.c b/drivers/staging/greybus/uart.c index 24b4dab069c3..172cf900eaf0 100644 --- a/drivers/staging/greybus/uart.c +++ b/drivers/staging/greybus/uart.c @@ -948,7 +948,8 @@ static int gb_tty_init(void) int retval = 0; gb_tty_driver = tty_alloc_driver(GB_NUM_MINORS, TTY_DRIVER_REAL_RAW | - TTY_DRIVER_DYNAMIC_DEV); + TTY_DRIVER_DYNAMIC_DEV | + TTY_DRIVER_RESET_SAVED_TERMIOS); if (IS_ERR(gb_tty_driver)) { pr_err("Can not allocate tty driver\n"); retval = PTR_ERR(gb_tty_driver); diff --git a/drivers/thermal/gov_step_wise.c b/drivers/thermal/gov_step_wise.c index ea277c466d8d..4fa4377f0d42 100644 --- a/drivers/thermal/gov_step_wise.c +++ b/drivers/thermal/gov_step_wise.c @@ -65,14 +65,12 @@ static unsigned long get_target_state(struct thermal_instance *instance, min(instance->lower + 1, instance->upper), instance->upper); } else if (trend == THERMAL_TREND_DROPPING) { - if (cur_state <= instance->lower) - return THERMAL_NO_TARGET; - /* - * If 'throttle' is false, no mitigation is necessary, so - * request the lower state for this instance. + * If 'throttle' is false, no mitigation is necessary and + * passive polling is already deactivated, so clear this + * instance state by returning THERMAL_NO_TARGET. */ - return instance->lower; + return THERMAL_NO_TARGET; } return instance->target; diff --git a/drivers/thunderbolt/ctl.c b/drivers/thunderbolt/ctl.c index cd47b627f97b..965988b18608 100644 --- a/drivers/thunderbolt/ctl.c +++ b/drivers/thunderbolt/ctl.c @@ -73,7 +73,6 @@ struct tb_ctl { #define tb_ctl_dbg_once(ctl, format, arg...) \ dev_dbg_once((ctl)->nhi->dev, format, ## arg) -static DECLARE_WAIT_QUEUE_HEAD(tb_cfg_request_cancel_queue); /* Serializes access to request kref_get/put */ static DEFINE_MUTEX(tb_cfg_request_lock); @@ -133,41 +132,42 @@ void tb_cfg_request_put(struct tb_cfg_request *req) static int tb_cfg_request_enqueue(struct tb_ctl *ctl, struct tb_cfg_request *req) { + tb_cfg_request_get(req); + WARN_ON(test_bit(TB_CFG_REQUEST_ACTIVE, &req->flags)); WARN_ON(req->ctl); - mutex_lock(&ctl->request_queue_lock); + guard(mutex)(&ctl->request_queue_lock); if (!ctl->running) { - mutex_unlock(&ctl->request_queue_lock); + tb_cfg_request_put(req); return -ENOTCONN; } req->ctl = ctl; list_add_tail(&req->list, &ctl->request_queue); set_bit(TB_CFG_REQUEST_ACTIVE, &req->flags); - mutex_unlock(&ctl->request_queue_lock); return 0; } +static bool tb_cfg_request_is_active(struct tb_cfg_request *req) +{ + return test_bit(TB_CFG_REQUEST_ACTIVE, &req->flags); +} + +static bool tb_cfg_request_is_canceled(struct tb_cfg_request *req) +{ + return test_bit(TB_CFG_REQUEST_CANCELED, &req->flags); +} + static void tb_cfg_request_dequeue(struct tb_cfg_request *req) { struct tb_ctl *ctl = req->ctl; - mutex_lock(&ctl->request_queue_lock); - if (!test_bit(TB_CFG_REQUEST_ACTIVE, &req->flags)) { - mutex_unlock(&ctl->request_queue_lock); - return; + guard(mutex)(&ctl->request_queue_lock); + if (tb_cfg_request_is_active(req)) { + list_del(&req->list); + clear_bit(TB_CFG_REQUEST_ACTIVE, &req->flags); + tb_cfg_request_put(req); } - - list_del(&req->list); - clear_bit(TB_CFG_REQUEST_ACTIVE, &req->flags); - if (test_bit(TB_CFG_REQUEST_CANCELED, &req->flags)) - wake_up(&tb_cfg_request_cancel_queue); - mutex_unlock(&ctl->request_queue_lock); -} - -static bool tb_cfg_request_is_active(struct tb_cfg_request *req) -{ - return test_bit(TB_CFG_REQUEST_ACTIVE, &req->flags); } static struct tb_cfg_request * @@ -178,7 +178,7 @@ tb_cfg_request_find(struct tb_ctl *ctl, struct ctl_pkg *pkg) mutex_lock(&pkg->ctl->request_queue_lock); list_for_each_entry(iter, &pkg->ctl->request_queue, list) { tb_cfg_request_get(iter); - if (iter->match(iter, pkg)) { + if (!tb_cfg_request_is_canceled(iter) && iter->match(iter, pkg)) { req = iter; break; } @@ -512,8 +512,11 @@ static void tb_ctl_rx_callback(struct tb_ring *ring, struct ring_frame *frame, trace_tb_rx(pkg->ctl->index, frame->eof, pkg->buffer, frame->size, !req); if (req) { - if (req->copy(req, pkg)) - schedule_work(&req->work); + scoped_guard(mutex, &pkg->ctl->request_queue_lock) { + if (!tb_cfg_request_is_canceled(req) && + req->copy(req, pkg)) + schedule_work(&req->work); + } tb_cfg_request_put(req); } @@ -525,11 +528,10 @@ static void tb_cfg_request_work(struct work_struct *work) { struct tb_cfg_request *req = container_of(work, typeof(*req), work); - if (!test_bit(TB_CFG_REQUEST_CANCELED, &req->flags)) + if (!tb_cfg_request_is_canceled(req)) req->callback(req->callback_data); tb_cfg_request_dequeue(req); - tb_cfg_request_put(req); } /** @@ -555,10 +557,9 @@ int tb_cfg_request(struct tb_ctl *ctl, struct tb_cfg_request *req, INIT_WORK(&req->work, tb_cfg_request_work); INIT_LIST_HEAD(&req->list); - tb_cfg_request_get(req); ret = tb_cfg_request_enqueue(ctl, req); if (ret) - goto err_put; + return ret; ret = tb_ctl_tx(ctl, req->request, req->request_size, req->request_type); @@ -572,9 +573,6 @@ int tb_cfg_request(struct tb_ctl *ctl, struct tb_cfg_request *req, err_dequeue: tb_cfg_request_dequeue(req); -err_put: - tb_cfg_request_put(req); - return ret; } @@ -588,9 +586,10 @@ int tb_cfg_request(struct tb_ctl *ctl, struct tb_cfg_request *req, */ void tb_cfg_request_cancel(struct tb_cfg_request *req, int err) { - set_bit(TB_CFG_REQUEST_CANCELED, &req->flags); - schedule_work(&req->work); - wait_event(tb_cfg_request_cancel_queue, !tb_cfg_request_is_active(req)); + scoped_guard(mutex, &req->ctl->request_queue_lock) + set_bit(TB_CFG_REQUEST_CANCELED, &req->flags); + cancel_work_sync(&req->work); + tb_cfg_request_dequeue(req); req->result.err = err; } diff --git a/drivers/thunderbolt/domain.c b/drivers/thunderbolt/domain.c index 12c88509a54f..24611f05b3cd 100644 --- a/drivers/thunderbolt/domain.c +++ b/drivers/thunderbolt/domain.c @@ -788,21 +788,6 @@ int tb_domain_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd, transmit_ring, receive_path, receive_ring); } -static void tb_domain_reset_interface(struct tb *tb) -{ - struct tb_nhi *nhi = tb->nhi; - - if (!nhi->ops->reset_interface) - return; - - guard(mutex)(&tb->lock); - - /* The reset clears the ring state so stop the control channel */ - tb_ctl_stop(tb->ctl); - nhi->ops->reset_interface(nhi); - tb_ctl_start(tb->ctl); -} - /** * tb_domain_disconnect_xdomain_paths() - Disable DMA paths for XDomain * @tb: Domain disabling the DMA paths @@ -825,20 +810,11 @@ int tb_domain_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd, int transmit_path, int transmit_ring, int receive_path, int receive_ring) { - int ret; - if (!tb->cm_ops->disconnect_xdomain_paths) return -ENOTSUPP; - ret = tb->cm_ops->disconnect_xdomain_paths(tb, xd, transmit_path, + return tb->cm_ops->disconnect_xdomain_paths(tb, xd, transmit_path, transmit_ring, receive_path, receive_ring); - if (ret) - return ret; - - if (tb->nhi->quirks & QUIRK_RESET_DMA_ON_TEARDOWN) - tb_domain_reset_interface(tb); - - return 0; } static int disconnect_xdomain(struct device *dev, void *data) diff --git a/drivers/thunderbolt/nhi.c b/drivers/thunderbolt/nhi.c index 5809809f64d4..be18f7b6595a 100644 --- a/drivers/thunderbolt/nhi.c +++ b/drivers/thunderbolt/nhi.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -560,6 +561,8 @@ static struct tb_ring *tb_ring_alloc(struct tb_nhi *nhi, u32 hop, int size, INIT_LIST_HEAD(&ring->in_flight); INIT_WORK(&ring->work, ring_work); init_waitqueue_head(&ring->wait); + lockdep_register_key(&ring->lock_key); + lockdep_init_map(&ring->work.lockdep_map, "ring.work", &ring->lock_key, 0); ring->nhi = nhi; ring->hop = hop; @@ -599,6 +602,7 @@ static struct tb_ring *tb_ring_alloc(struct tb_nhi *nhi, u32 hop, int size, ring->size * sizeof(*ring->descriptors), ring->descriptors, ring->descriptors_dma); err_free_ring: + lockdep_unregister_key(&ring->lock_key); kfree(ring); return NULL; @@ -848,6 +852,7 @@ void tb_ring_free(struct tb_ring *ring) * to finish before freeing the ring. */ flush_work(&ring->work); + lockdep_unregister_key(&ring->lock_key); kfree(ring); } EXPORT_SYMBOL_GPL(tb_ring_free); @@ -1175,32 +1180,6 @@ static void nhi_reset(struct tb_nhi *nhi) dev_warn(nhi->dev, "timeout resetting host router\n"); } -/** - * nhi_reset_interface() - Reset the host interface - * @nhi: Host interface to reset - * - * Brings the registers in the memory BAR back to their default state and - * clears the End-to-End Flow Control state. The caller is responsible for - * stopping the control channel over the reset because it clears the ring - * state as well. - */ -void nhi_reset_interface(struct tb_nhi *nhi) -{ - u32 val; - - val = ioread32(nhi->iobase + REG_CAPS); - /* Only v1 host interfaces implement the reset */ - if (FIELD_GET(REG_CAPS_VERSION_MASK, val) >= REG_CAPS_VERSION_2) - return; - - dev_dbg(nhi->dev, "issuing host interface reset\n"); - - iowrite32(REG_HOST_INTERFACE_RESET_RST, - nhi->iobase + REG_HOST_INTERFACE_RESET); - /* Wait for tHIReset (10 ms) to complete */ - usleep_range(10000, 20000); -} - static struct tb *nhi_select_cm(struct tb_nhi *nhi) { struct tb *tb; diff --git a/drivers/thunderbolt/nhi.h b/drivers/thunderbolt/nhi.h index f72d6b274501..d488eadadfce 100644 --- a/drivers/thunderbolt/nhi.h +++ b/drivers/thunderbolt/nhi.h @@ -36,8 +36,6 @@ irqreturn_t nhi_msi(int irq, void *data); irqreturn_t ring_msix(int irq, void *data); int nhi_probe(struct tb_nhi *nhi); void nhi_shutdown(struct tb_nhi *nhi); -void nhi_reset_interface(struct tb_nhi *nhi); - extern const struct dev_pm_ops nhi_pm_ops; /** @@ -54,7 +52,6 @@ extern const struct dev_pm_ops nhi_pm_ops; * @release_ring_irq: NHI specific interrupt release hook * @is_present: Whether the device is currently present on the parent bus * @init_interrupts: NHI specific interrupt initialization hook - * @reset_interface: Resets the host interface */ struct tb_nhi_ops { int (*init)(struct tb_nhi *nhi); @@ -69,7 +66,6 @@ struct tb_nhi_ops { void (*release_ring_irq)(struct tb_ring *ring); bool (*is_present)(struct tb_nhi *nhi); int (*init_interrupts)(struct tb_nhi *nhi); - void (*reset_interface)(struct tb_nhi *nhi); }; /* @@ -120,24 +116,11 @@ struct tb_nhi_ops { #define PCI_DEVICE_ID_INTEL_PTL_P_NHI0 0xe433 #define PCI_DEVICE_ID_INTEL_PTL_P_NHI1 0xe434 -#define PCI_DEVICE_ID_AMD_1AH_M60H_NHI0 0x1120 -#define PCI_DEVICE_ID_AMD_1AH_M60H_NHI1 0x1121 -#define PCI_DEVICE_ID_AMD_1AH_M68H_NHI0 0x113b -#define PCI_DEVICE_ID_AMD_1AH_M68H_NHI1 0x113c -#define PCI_DEVICE_ID_AMD_1AH_M80H_NHI0 0x1155 -#define PCI_DEVICE_ID_AMD_1AH_M80H_NHI1 0x1158 -#define PCI_DEVICE_ID_AMD_1AH_M80H_NHI2 0x1159 -#define PCI_DEVICE_ID_AMD_1AH_M24H_NHI0 0x151c -#define PCI_DEVICE_ID_AMD_1AH_M24H_NHI1 0x151d -#define PCI_DEVICE_ID_AMD_1AH_M70H_NHI0 0x158d -#define PCI_DEVICE_ID_AMD_1AH_M70H_NHI1 0x158e - #define PCI_CLASS_SERIAL_USB_USB4 0x0c0340 /* Host interface quirks */ -#define QUIRK_AUTO_CLEAR_INT BIT(0) -#define QUIRK_E2E BIT(1) -#define QUIRK_RESET_DMA_ON_TEARDOWN BIT(2) +#define QUIRK_AUTO_CLEAR_INT BIT(0) +#define QUIRK_E2E BIT(1) /* * Minimal number of vectors when we use MSI-X. Two for control channel diff --git a/drivers/thunderbolt/nhi_regs.h b/drivers/thunderbolt/nhi_regs.h index 99df60b6db36..d6a197fabc74 100644 --- a/drivers/thunderbolt/nhi_regs.h +++ b/drivers/thunderbolt/nhi_regs.h @@ -115,10 +115,6 @@ struct ring_desc { #define REG_CAPS_VERSION_MASK GENMASK(23, 16) #define REG_CAPS_VERSION_2 0x40 -/* Host Interface Reset - resets TX/RX rings and E2E flow control counters */ -#define REG_HOST_INTERFACE_RESET 0x39858 -#define REG_HOST_INTERFACE_RESET_RST BIT(0) - #define REG_DMA_MISC 0x39864 #define REG_DMA_MISC_INT_AUTO_CLEAR BIT(2) #define REG_DMA_MISC_DISABLE_AUTO_CLEAR BIT(17) diff --git a/drivers/thunderbolt/pci.c b/drivers/thunderbolt/pci.c index 99333729f3c2..8462ccb59b7e 100644 --- a/drivers/thunderbolt/pci.c +++ b/drivers/thunderbolt/pci.c @@ -62,27 +62,6 @@ static void nhi_pci_check_quirks(struct tb_nhi_pci *nhi_pci) nhi->quirks |= QUIRK_E2E; break; } - } else if (pdev->vendor == PCI_VENDOR_ID_AMD) { - switch (pdev->device) { - case PCI_DEVICE_ID_AMD_1AH_M60H_NHI0: - case PCI_DEVICE_ID_AMD_1AH_M60H_NHI1: - case PCI_DEVICE_ID_AMD_1AH_M68H_NHI0: - case PCI_DEVICE_ID_AMD_1AH_M68H_NHI1: - case PCI_DEVICE_ID_AMD_1AH_M80H_NHI0: - case PCI_DEVICE_ID_AMD_1AH_M80H_NHI1: - case PCI_DEVICE_ID_AMD_1AH_M80H_NHI2: - case PCI_DEVICE_ID_AMD_1AH_M24H_NHI0: - case PCI_DEVICE_ID_AMD_1AH_M24H_NHI1: - case PCI_DEVICE_ID_AMD_1AH_M70H_NHI0: - case PCI_DEVICE_ID_AMD_1AH_M70H_NHI1: - /* - * These AMD hosts may hang the Tx ring when the - * DMA paths are torn down so they need the host - * interface reset after each teardown. - */ - nhi->quirks |= QUIRK_RESET_DMA_ON_TEARDOWN; - break; - } } } @@ -279,7 +258,6 @@ static const struct tb_nhi_ops pci_nhi_default_ops = { .shutdown = nhi_pci_release_irq, .is_present = nhi_pci_is_present, .init_interrupts = nhi_pci_init_msi, - .reset_interface = nhi_reset_interface, }; /* Ice Lake specific NHI operations */ @@ -463,7 +441,6 @@ static const struct tb_nhi_ops icl_nhi_ops = { .release_ring_irq = nhi_pci_ring_release_msix, .is_present = nhi_pci_is_present, .init_interrupts = nhi_pci_init_msi, - .reset_interface = nhi_reset_interface, }; static int nhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) diff --git a/drivers/thunderbolt/quirks.c b/drivers/thunderbolt/quirks.c index 9f7914ac2f48..96b7fe0386e3 100644 --- a/drivers/thunderbolt/quirks.c +++ b/drivers/thunderbolt/quirks.c @@ -99,6 +99,9 @@ static const struct tb_quirk tb_quirks[] = { quirk_usb3_maximum_bandwidth }, { 0x8087, PCI_DEVICE_ID_INTEL_BARLOW_RIDGE_HUB_40G_BRIDGE, 0x0000, 0x0000, quirk_usb3_maximum_bandwidth }, + /* CLx is unstable on the Anker Prime TB5 dock (DROM 01ea:83b5) */ + { 0x8087, PCI_DEVICE_ID_INTEL_BARLOW_RIDGE_HUB_80G_BRIDGE, 0x01ea, 0x83b5, + quirk_clx_disable }, /* * Block Runtime PM in DP redrive mode for Intel Barlow Ridge host * controllers. diff --git a/drivers/thunderbolt/stream.c b/drivers/thunderbolt/stream.c index 25c259dd0760..988765c98c8c 100644 --- a/drivers/thunderbolt/stream.c +++ b/drivers/thunderbolt/stream.c @@ -978,6 +978,9 @@ static int tbstream_dev_fops_open(struct inode *inode, struct file *file) } mutex_unlock(&sdev->lock); + + /* Stream handles IOCB_NOWAIT just fine */ + file->f_mode |= FMODE_NOWAIT; return 0; err_unlock: diff --git a/drivers/thunderbolt/switch.c b/drivers/thunderbolt/switch.c index 404c0693df50..cf571a7c98d0 100644 --- a/drivers/thunderbolt/switch.c +++ b/drivers/thunderbolt/switch.c @@ -774,6 +774,7 @@ static int tb_port_alloc_hopid(struct tb_port *port, bool in, int min_hopid, { int port_max_hopid; struct ida *ida; + int ret; if (in) { port_max_hopid = port->config.max_in_hop_id; @@ -793,7 +794,11 @@ static int tb_port_alloc_hopid(struct tb_port *port, bool in, int min_hopid, if (max_hopid < 0 || max_hopid > port_max_hopid) max_hopid = port_max_hopid; - return ida_alloc_range(ida, min_hopid, max_hopid, GFP_KERNEL); + ret = ida_alloc_range(ida, min_hopid, max_hopid, GFP_KERNEL); + if (ret >= 0) + tb_switch_get(port->sw); + + return ret; } /** @@ -832,6 +837,7 @@ int tb_port_alloc_out_hopid(struct tb_port *port, int min_hopid, int max_hopid) void tb_port_release_in_hopid(struct tb_port *port, int hopid) { ida_free(&port->in_hopids, hopid); + tb_switch_put(port->sw); } /** @@ -842,6 +848,7 @@ void tb_port_release_in_hopid(struct tb_port *port, int hopid) void tb_port_release_out_hopid(struct tb_port *port, int hopid) { ida_free(&port->out_hopids, hopid); + tb_switch_put(port->sw); } static inline bool tb_switch_is_reachable(const struct tb_switch *parent, diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c index 47753a5c0f2e..4e5d0578fd4b 100644 --- a/drivers/thunderbolt/tb.c +++ b/drivers/thunderbolt/tb.c @@ -89,6 +89,7 @@ static void tb_dp_resource_unavailable(struct tb *tb, struct tb_port *port, const char *reason); static void tb_queue_dp_bandwidth_request(struct tb *tb, u64 route, u8 port, int retry, unsigned long delay); +static void tb_dp_tunnel_active(struct tb_tunnel *tunnel, void *data); static void tb_queue_hotplug(struct tb *tb, u64 route, u8 port, bool unplug) { @@ -385,7 +386,8 @@ static void tb_switch_discover_tunnels(struct tb_switch *sw, switch (port->config.type) { case TB_TYPE_DP_HDMI_IN: - tunnel = tb_tunnel_discover_dp(tb, port, alloc_hopids); + tunnel = tb_tunnel_discover_dp(tb, port, alloc_hopids, + tb_dp_tunnel_active, tb); tb_increase_tmu_accuracy(tunnel); break; @@ -1910,6 +1912,18 @@ static void tb_dp_tunnel_active(struct tb_tunnel *tunnel, void *data) struct tb *tb = data; mutex_lock(&tb->lock); + + /* + * If the DPRX read was canceled the tunnel is already being torn + * down by whoever canceled it. Do not touch the adapters here + * because the routers may be gone by now. + */ + if (tunnel->dprx_canceled) { + tb_tunnel_dbg(tunnel, "DPRX read canceled, not activating\n"); + mutex_unlock(&tb->lock); + return; + } + if (tb_tunnel_is_active(tunnel)) { int consumed_up, consumed_down, ret; @@ -1964,8 +1978,6 @@ static void tb_dp_tunnel_active(struct tb_tunnel *tunnel, void *data) tb_dp_resource_unavailable(tb, in, "DPRX negotiation failed"); } mutex_unlock(&tb->lock); - - tb_domain_put(tb); } static void tb_tunnel_one_dp(struct tb *tb, struct tb_port *in, @@ -2026,8 +2038,7 @@ static void tb_tunnel_one_dp(struct tb *tb, struct tb_port *in, available_up, available_down); tunnel = tb_tunnel_alloc_dp(tb, in, out, link_nr, available_up, - available_down, tb_dp_tunnel_active, - tb_domain_get(tb)); + available_down, tb_dp_tunnel_active, tb); if (!tunnel) { tb_port_dbg(out, "could not allocate DP tunnel\n"); goto err_reclaim_usb; @@ -2048,7 +2059,6 @@ static void tb_tunnel_one_dp(struct tb *tb, struct tb_port *in, tb_tunnel_put(tunnel); err_reclaim_usb: tb_reclaim_usb3_bandwidth(tb, in, out); - tb_domain_put(tb); err_detach_group: tb_detach_bandwidth_group(in); err_dealloc_dp: @@ -2950,11 +2960,12 @@ static void tb_stop(struct tb *tb) /* tunnels are only present after everything has been initialized */ list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) { /* - * DMA tunnels require the driver to be functional so we - * tear them down. Other protocol tunnels can be left - * intact. + * DMA tunnels and DP tunnels which are not yet active require + * the driver to be functional so we tear them down. + * Other protocol tunnels can be left intact. */ - if (tb_tunnel_is_dma(tunnel)) + if (tb_tunnel_is_dma(tunnel) || + (tb_tunnel_is_dp(tunnel) && !tb_tunnel_is_active(tunnel))) tb_tunnel_deactivate(tunnel); tb_tunnel_put(tunnel); } @@ -3274,11 +3285,11 @@ static void tb_remove_work(struct work_struct *work) struct tb *tb = tcm_to_tb(tcm); mutex_lock(&tb->lock); - if (tb->root_switch) + if (tb->root_switch) { tb_free_unplugged_children(tb->root_switch); + tb_free_unplugged_xdomains(tb->root_switch); + } mutex_unlock(&tb->lock); - - tb_free_unplugged_xdomains(tb->root_switch); } static int tb_runtime_resume(struct tb *tb) diff --git a/drivers/thunderbolt/test.c b/drivers/thunderbolt/test.c index 05652ee82fbf..c8c648f31107 100644 --- a/drivers/thunderbolt/test.c +++ b/drivers/thunderbolt/test.c @@ -33,6 +33,18 @@ static void kunit_ida_init(struct kunit *test, struct ida *ida) kunit_alloc_resource(test, __ida_init, __ida_destroy, GFP_KERNEL, ida); } +static void tb_test_switch_release(struct device *dev) +{ + /* The memory is owned by KUnit, nothing to do here */ +} + +static void tb_test_switch_put(void *data) +{ + struct tb_switch *sw = data; + + put_device(&sw->dev); +} + static struct tb_switch *alloc_switch(struct kunit *test, u64 route, u8 upstream_port, u8 max_port_number) { @@ -44,6 +56,15 @@ static struct tb_switch *alloc_switch(struct kunit *test, u64 route, if (!sw) return NULL; + /* + * HopID allocations take a reference to their routers and those devices + * have to be initialized for that to work. + */ + sw->dev.release = tb_test_switch_release; + device_initialize(&sw->dev); + if (kunit_add_action_or_reset(test, tb_test_switch_put, sw)) + return NULL; + sw->config.upstream_port_number = upstream_port; sw->config.depth = tb_route_length(route); sw->config.route_hi = upper_32_bits(route); @@ -1386,6 +1407,10 @@ static void tb_test_tunnel_pcie(struct kunit *test) tb_tunnel_put(tunnel1); } +static void tb_test_dp_tunnel_active(struct tb_tunnel *tunnel, void *data) +{ +} + static void tb_test_tunnel_dp(struct kunit *test) { struct tb_switch *host, *dev; @@ -1406,7 +1431,8 @@ static void tb_test_tunnel_dp(struct kunit *test) in = &host->ports[5]; out = &dev->ports[13]; - tunnel = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0, NULL, NULL); + tunnel = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0, + tb_test_dp_tunnel_active, NULL); KUNIT_ASSERT_NOT_NULL(test, tunnel); KUNIT_EXPECT_EQ(test, tunnel->type, TB_TUNNEL_DP); KUNIT_EXPECT_PTR_EQ(test, tunnel->src_port, in); @@ -1452,7 +1478,8 @@ static void tb_test_tunnel_dp_chain(struct kunit *test) in = &host->ports[5]; out = &dev4->ports[14]; - tunnel = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0, NULL, NULL); + tunnel = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0, + tb_test_dp_tunnel_active, NULL); KUNIT_ASSERT_NOT_NULL(test, tunnel); KUNIT_EXPECT_EQ(test, tunnel->type, TB_TUNNEL_DP); KUNIT_EXPECT_PTR_EQ(test, tunnel->src_port, in); @@ -1502,7 +1529,8 @@ static void tb_test_tunnel_dp_tree(struct kunit *test) in = &dev2->ports[13]; out = &dev5->ports[13]; - tunnel = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0, NULL, NULL); + tunnel = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0, + tb_test_dp_tunnel_active, NULL); KUNIT_ASSERT_NOT_NULL(test, tunnel); KUNIT_EXPECT_EQ(test, tunnel->type, TB_TUNNEL_DP); KUNIT_EXPECT_PTR_EQ(test, tunnel->src_port, in); @@ -1567,7 +1595,8 @@ static void tb_test_tunnel_dp_max_length(struct kunit *test) in = &dev6->ports[13]; out = &dev12->ports[13]; - tunnel = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0, NULL, NULL); + tunnel = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0, + tb_test_dp_tunnel_active, NULL); KUNIT_ASSERT_NOT_NULL(test, tunnel); KUNIT_EXPECT_EQ(test, tunnel->type, TB_TUNNEL_DP); KUNIT_EXPECT_PTR_EQ(test, tunnel->src_port, in); @@ -1637,7 +1666,8 @@ static void tb_test_tunnel_3dp(struct kunit *test) out2 = &dev5->ports[13]; out3 = &dev4->ports[14]; - tunnel1 = tb_tunnel_alloc_dp(NULL, in1, out1, 1, 0, 0, NULL, NULL); + tunnel1 = tb_tunnel_alloc_dp(NULL, in1, out1, 1, 0, 0, + tb_test_dp_tunnel_active, NULL); KUNIT_ASSERT_TRUE(test, tunnel1 != NULL); KUNIT_EXPECT_EQ(test, tunnel1->type, TB_TUNNEL_DP); KUNIT_EXPECT_PTR_EQ(test, tunnel1->src_port, in1); @@ -1645,7 +1675,8 @@ static void tb_test_tunnel_3dp(struct kunit *test) KUNIT_ASSERT_EQ(test, tunnel1->npaths, 3); KUNIT_ASSERT_EQ(test, tunnel1->paths[0]->path_length, 3); - tunnel2 = tb_tunnel_alloc_dp(NULL, in2, out2, 1, 0, 0, NULL, NULL); + tunnel2 = tb_tunnel_alloc_dp(NULL, in2, out2, 1, 0, 0, + tb_test_dp_tunnel_active, NULL); KUNIT_ASSERT_TRUE(test, tunnel2 != NULL); KUNIT_EXPECT_EQ(test, tunnel2->type, TB_TUNNEL_DP); KUNIT_EXPECT_PTR_EQ(test, tunnel2->src_port, in2); @@ -1653,7 +1684,8 @@ static void tb_test_tunnel_3dp(struct kunit *test) KUNIT_ASSERT_EQ(test, tunnel2->npaths, 3); KUNIT_ASSERT_EQ(test, tunnel2->paths[0]->path_length, 4); - tunnel3 = tb_tunnel_alloc_dp(NULL, in3, out3, 1, 0, 0, NULL, NULL); + tunnel3 = tb_tunnel_alloc_dp(NULL, in3, out3, 1, 0, 0, + tb_test_dp_tunnel_active, NULL); KUNIT_ASSERT_TRUE(test, tunnel3 != NULL); KUNIT_EXPECT_EQ(test, tunnel3->type, TB_TUNNEL_DP); KUNIT_EXPECT_PTR_EQ(test, tunnel3->src_port, in3); @@ -1751,7 +1783,8 @@ static void tb_test_tunnel_port_on_path(struct kunit *test) in = &dev2->ports[13]; out = &dev5->ports[13]; - dp_tunnel = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0, NULL, NULL); + dp_tunnel = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0, + tb_test_dp_tunnel_active, NULL); KUNIT_ASSERT_NOT_NULL(test, dp_tunnel); KUNIT_EXPECT_TRUE(test, tb_tunnel_port_on_path(dp_tunnel, in)); @@ -2183,7 +2216,8 @@ static void tb_test_credit_alloc_dp(struct kunit *test) in = &host->ports[5]; out = &dev->ports[14]; - tunnel = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0, NULL, NULL); + tunnel = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0, + tb_test_dp_tunnel_active, NULL); KUNIT_ASSERT_NOT_NULL(test, tunnel); KUNIT_ASSERT_EQ(test, tunnel->npaths, (size_t)3); @@ -2419,7 +2453,8 @@ static struct tb_tunnel *TB_TEST_DP_TUNNEL1(struct kunit *test, in = &host->ports[5]; out = &dev->ports[13]; - dp_tunnel1 = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0, NULL, NULL); + dp_tunnel1 = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0, + tb_test_dp_tunnel_active, NULL); KUNIT_ASSERT_NOT_NULL(test, dp_tunnel1); KUNIT_ASSERT_EQ(test, dp_tunnel1->npaths, (size_t)3); @@ -2456,7 +2491,8 @@ static struct tb_tunnel *TB_TEST_DP_TUNNEL2(struct kunit *test, in = &host->ports[6]; out = &dev->ports[14]; - dp_tunnel2 = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0, NULL, NULL); + dp_tunnel2 = tb_tunnel_alloc_dp(NULL, in, out, 1, 0, 0, + tb_test_dp_tunnel_active, NULL); KUNIT_ASSERT_NOT_NULL(test, dp_tunnel2); KUNIT_ASSERT_EQ(test, dp_tunnel2->npaths, (size_t)3); diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c index 7e8284575dff..ffd2d04b3bfa 100644 --- a/drivers/thunderbolt/tunnel.c +++ b/drivers/thunderbolt/tunnel.c @@ -510,6 +510,7 @@ struct tb_tunnel *tb_tunnel_discover_pci(struct tb *tb, struct tb_port *down, goto err_deactivate; } + tb_tunnel_set_active(tunnel, true); tb_tunnel_dbg(tunnel, "discovered\n"); return tunnel; @@ -1093,8 +1094,14 @@ static void tb_dp_dprx_work(struct work_struct *work) struct tb_tunnel *tunnel = container_of(work, typeof(*tunnel), dprx_work.work); struct tb *tb = tunnel->tb; + /* + * The DPRX read can be canceled while this work is waiting for + * tb->lock. Check the flag only once it is held: while the lock is + * held the tunnel cannot be torn down under us and the adapters are + * safe to access. + */ + mutex_lock(&tb->lock); if (!tunnel->dprx_canceled) { - mutex_lock(&tb->lock); if (tb_dp_is_usb4(tunnel->src_port->sw) && tb_dp_wait_dprx(tunnel, TB_DPRX_WAIT_TIMEOUT)) { if (ktime_before(ktime_get(), tunnel->dprx_timeout)) { @@ -1106,41 +1113,42 @@ static void tb_dp_dprx_work(struct work_struct *work) } else { tb_tunnel_set_active(tunnel, true); } - mutex_unlock(&tb->lock); } + mutex_unlock(&tb->lock); - if (tunnel->callback) - tunnel->callback(tunnel, tunnel->callback_data); + tunnel->callback(tunnel, tunnel->callback_data); tb_tunnel_put(tunnel); + tb_domain_put(tb); } static int tb_dp_dprx_start(struct tb_tunnel *tunnel) { /* - * Bump up the reference to keep the tunnel around. It will be - * dropped in tb_dp_dprx_stop() once the tunnel is deactivated. + * Bump up the references to keep the tunnel and the domain around + * until the work has run or has been canceled. */ tb_tunnel_get(tunnel); + tb_domain_get(tunnel->tb); tunnel->dprx_started = true; + tunnel->dprx_canceled = false; + tunnel->dprx_timeout = dprx_timeout_to_ktime(dprx_timeout); + queue_delayed_work(tunnel->tb->wq, &tunnel->dprx_work, 0); - if (tunnel->callback) { - tunnel->dprx_timeout = dprx_timeout_to_ktime(dprx_timeout); - queue_delayed_work(tunnel->tb->wq, &tunnel->dprx_work, 0); - return -EINPROGRESS; - } - - return tb_dp_is_usb4(tunnel->src_port->sw) ? - tb_dp_wait_dprx(tunnel, dprx_timeout) : 0; + return -EINPROGRESS; } static void tb_dp_dprx_stop(struct tb_tunnel *tunnel) { + struct tb *tb = tunnel->tb; + if (tunnel->dprx_started) { tunnel->dprx_started = false; tunnel->dprx_canceled = true; - if (cancel_delayed_work(&tunnel->dprx_work)) + if (cancel_delayed_work(&tunnel->dprx_work)) { tb_tunnel_put(tunnel); + tb_domain_put(tb); + } } } @@ -1582,20 +1590,28 @@ static void tb_dp_dump(struct tb_tunnel *tunnel) * @tb: Pointer to the domain structure * @in: DP in adapter * @alloc_hopid: Allocate HopIDs from visited ports + * @callback: Callback that is called when the DP tunnel is fully + * activated (or there is an error) + * @callback_data: Data for @callback * * If @in adapter is active, follows the tunnel to the DP out adapter * and back. Returns the discovered tunnel or %NULL if there was no - * tunnel. + * tunnel. See tb_tunnel_alloc_dp() for @callback. * * Return: Pointer to &struct tb_tunnel or %NULL if no tunnel found. */ struct tb_tunnel *tb_tunnel_discover_dp(struct tb *tb, struct tb_port *in, - bool alloc_hopid) + bool alloc_hopid, + void (*callback)(struct tb_tunnel *, void *), + void *callback_data) { struct tb_tunnel *tunnel; struct tb_port *port; struct tb_path *path; + if (WARN_ON(!callback)) + return NULL; + if (!tb_dp_port_is_enabled(in)) return NULL; @@ -1611,6 +1627,9 @@ struct tb_tunnel *tb_tunnel_discover_dp(struct tb *tb, struct tb_port *in, tunnel->alloc_bandwidth = tb_dp_alloc_bandwidth; tunnel->consumed_bandwidth = tb_dp_consumed_bandwidth; tunnel->src_port = in; + tunnel->callback = callback; + tunnel->callback_data = callback_data; + INIT_DELAYED_WORK(&tunnel->dprx_work, tb_dp_dprx_work); path = tb_path_discover(in, TB_DP_VIDEO_HOPID, NULL, -1, &tunnel->dst_port, "Video", alloc_hopid); @@ -1656,6 +1675,7 @@ struct tb_tunnel *tb_tunnel_discover_dp(struct tb *tb, struct tb_port *in, tb_dp_dump(tunnel); + tb_tunnel_set_active(tunnel, true); tb_tunnel_dbg(tunnel, "discovered\n"); return tunnel; @@ -1677,16 +1697,16 @@ struct tb_tunnel *tb_tunnel_discover_dp(struct tb *tb, struct tb_port *in, * %0 if no available bandwidth. * @max_down: Maximum available downstream bandwidth for the DP tunnel. * %0 if no available bandwidth. - * @callback: Optional callback that is called when the DP tunnel is - * fully activated (or there is an error) - * @callback_data: Optional data for @callback + * @callback: Callback that is called when the DP tunnel is fully + * activated (or there is an error) + * @callback_data: Data for @callback * * Allocates a tunnel between @in and @out that is capable of tunneling - * Display Port traffic. If @callback is not %NULL it will be called - * after tb_tunnel_activate() once the tunnel has been fully activated. - * It can call tb_tunnel_is_active() to check if activation was - * successful (or if it returns %false there was some sort of issue). - * The @callback is called without @tb->lock held. + * Display Port traffic. The @callback is called after tb_tunnel_activate() + * once the tunnel has been fully activated. It can call + * tb_tunnel_is_active() to check if activation was successful (or if it + * returns %false there was some sort of issue). The @callback is called + * without @tb->lock held. * * Return: Pointer to @struct tb_tunnel or %NULL in case of failure. */ @@ -1701,7 +1721,7 @@ struct tb_tunnel *tb_tunnel_alloc_dp(struct tb *tb, struct tb_port *in, struct tb_path *path; bool pm_support; - if (WARN_ON(!in->cap_adap || !out->cap_adap)) + if (WARN_ON(!in->cap_adap || !out->cap_adap || !callback)) return NULL; tunnel = tb_tunnel_alloc(tb, 3, TB_TUNNEL_DP); @@ -2288,6 +2308,7 @@ struct tb_tunnel *tb_tunnel_discover_usb3(struct tb *tb, struct tb_port *down, tb_usb3_reclaim_available_bandwidth; } + tb_tunnel_set_active(tunnel, true); tb_tunnel_dbg(tunnel, "discovered\n"); return tunnel; diff --git a/drivers/thunderbolt/tunnel.h b/drivers/thunderbolt/tunnel.h index 4878763a82b3..7d1d255ab5a7 100644 --- a/drivers/thunderbolt/tunnel.h +++ b/drivers/thunderbolt/tunnel.h @@ -66,8 +66,8 @@ enum tb_tunnel_state { * @dprx_canceled: Was DPRX capabilities read poll canceled * @dprx_timeout: If set DPRX capabilities read poll work will timeout after this passes * @dprx_work: Worker that is scheduled to poll completion of DPRX capabilities read - * @callback: Optional callback called when DP tunnel is fully activated - * @callback_data: Optional data for @callback + * @callback: Callback called when DP tunnel is fully activated + * @callback_data: Data for @callback * @paths: All paths required by the tunnel */ struct tb_tunnel { @@ -117,7 +117,9 @@ struct tb_tunnel *tb_tunnel_alloc_pci(struct tb *tb, struct tb_port *up, bool tb_tunnel_reserved_pci(struct tb_port *port, int *reserved_up, int *reserved_down); struct tb_tunnel *tb_tunnel_discover_dp(struct tb *tb, struct tb_port *in, - bool alloc_hopid); + bool alloc_hopid, + void (*callback)(struct tb_tunnel *, void *), + void *callback_data); struct tb_tunnel *tb_tunnel_alloc_dp(struct tb *tb, struct tb_port *in, struct tb_port *out, int link_nr, int max_up, int max_down, diff --git a/drivers/thunderbolt/xdomain.c b/drivers/thunderbolt/xdomain.c index c179bd751fe4..b187a60663f5 100644 --- a/drivers/thunderbolt/xdomain.c +++ b/drivers/thunderbolt/xdomain.c @@ -377,6 +377,10 @@ static int tb_xdp_properties_request(struct tb_ctl *ctl, u64 route, len += sizeof(res->hdr.xd_hdr) / 4; len -= sizeof(*res) / 4; + if (len > TB_XDP_PROPERTIES_MAX_DATA_LENGTH) { + ret = -EINVAL; + goto err; + } if (res->offset != req.offset) { ret = -EINVAL; @@ -1814,7 +1818,6 @@ static void tb_xdomain_state_work(struct work_struct *work) tb_xdomain_failed(xd); } else { xd->state = XDOMAIN_STATE_ENUMERATED; - tb_xdomain_queue_properties_changed(xd); } break; diff --git a/drivers/tty/amiserial.c b/drivers/tty/amiserial.c index 28af0fd98181..60f6060b893d 100644 --- a/drivers/tty/amiserial.c +++ b/drivers/tty/amiserial.c @@ -1138,11 +1138,6 @@ static int rs_ioctl(struct tty_struct *tty, local_irq_save(flags); cnow = info->icount; /* atomic copy */ local_irq_restore(flags); - if (cnow.rng == cprev.rng && cnow.dsr == cprev.dsr && - cnow.dcd == cprev.dcd && cnow.cts == cprev.cts) { - ret = -EIO; /* no change => error */ - break; - } if ( ((arg & TIOCM_RNG) && (cnow.rng != cprev.rng)) || ((arg & TIOCM_DSR) && (cnow.dsr != cprev.dsr)) || ((arg & TIOCM_CD) && (cnow.dcd != cprev.dcd)) || @@ -1150,6 +1145,10 @@ static int rs_ioctl(struct tty_struct *tty, ret = 0; break; } + if (tty_io_error(tty)) { + ret = -EIO; + break; + } schedule(); /* see if a signal did it */ if (signal_pending(current)) { diff --git a/drivers/tty/n_gsm.c b/drivers/tty/n_gsm.c index c13e050de83b..e55779ff0af2 100644 --- a/drivers/tty/n_gsm.c +++ b/drivers/tty/n_gsm.c @@ -454,14 +454,13 @@ static const u8 gsm_fcs8[256] = { static void gsm_dlci_close(struct gsm_dlci *dlci); static int gsmld_output(struct gsm_mux *gsm, u8 *data, int len); -static int gsm_modem_update(struct gsm_dlci *dlci, u8 brk); +static int gsm_modem_update(struct gsm_dlci *dlci, u8 brk, bool wait); static struct gsm_msg *gsm_data_alloc(struct gsm_mux *gsm, u8 addr, int len, u8 ctrl); static int gsm_send_packet(struct gsm_mux *gsm, struct gsm_msg *msg); static struct gsm_dlci *gsm_dlci_alloc(struct gsm_mux *gsm, int addr); static void gsmld_write_trigger(struct gsm_mux *gsm); static void gsmld_write_task(struct work_struct *work); -static int gsm_modem_send_initial_msc(struct gsm_dlci *dlci); /** * gsm_fcs_add - update FCS @@ -2174,7 +2173,7 @@ static void gsm_dlci_open(struct gsm_dlci *dlci) pr_debug("DLCI %d goes open.\n", dlci->addr); /* Send current modem state */ if (dlci->addr) { - gsm_modem_send_initial_msc(dlci); + gsm_modem_update(dlci, 0, false); } else { /* Start keep-alive control */ gsm->ka_num = 0; @@ -4136,9 +4135,10 @@ static void gsm_modem_upd_via_data(struct gsm_dlci *dlci, u8 brk) * gsm_modem_upd_via_msc - send modem bits via control frame * @dlci: channel * @brk: break signal + * @wait: wait for the MSC command */ -static int gsm_modem_upd_via_msc(struct gsm_dlci *dlci, u8 brk) +static int gsm_modem_upd_via_msc(struct gsm_dlci *dlci, u8 brk, bool wait) { u8 modembits[3]; struct gsm_control *ctrl; @@ -4155,41 +4155,22 @@ static int gsm_modem_upd_via_msc(struct gsm_dlci *dlci, u8 brk) modembits[2] = (brk << 4) | 2 | EA; /* Length, Break, EA */ len++; } + if (!wait) + return gsm_control_command(dlci->gsm, CMD_MSC, modembits, len); ctrl = gsm_control_send(dlci->gsm, CMD_MSC, modembits, len); if (ctrl == NULL) return -ENOMEM; return gsm_control_wait(dlci->gsm, ctrl); } -/** - * gsm_modem_send_initial_msc - Send initial modem status message - * - * @dlci: channel - * - * Send an initial MSC message after DLCI open to set the initial - * modem status lines. This is only done for basic mode. - * Does not wait for a response as we cannot block the input queue - * processing. - */ -static int gsm_modem_send_initial_msc(struct gsm_dlci *dlci) -{ - u8 modembits[2]; - - if (dlci->adaption != 1 || dlci->gsm->encoding != GSM_BASIC_OPT) - return 0; - - modembits[0] = (dlci->addr << 2) | 2 | EA; /* DLCI, Valid, EA */ - modembits[1] = (gsm_encode_modem(dlci) << 1) | EA; - return gsm_control_command(dlci->gsm, CMD_MSC, (const u8 *)&modembits, 2); -} - /** * gsm_modem_update - send modem status line state * @dlci: channel * @brk: break signal + * @wait: wait for the MSC command */ -static int gsm_modem_update(struct gsm_dlci *dlci, u8 brk) +static int gsm_modem_update(struct gsm_dlci *dlci, u8 brk, bool wait) { if (dlci->gsm->dead) return -EL2HLT; @@ -4199,7 +4180,7 @@ static int gsm_modem_update(struct gsm_dlci *dlci, u8 brk) return 0; } else if (dlci->gsm->encoding == GSM_BASIC_OPT) { /* Send as MSC control message. */ - return gsm_modem_upd_via_msc(dlci, brk); + return gsm_modem_upd_via_msc(dlci, brk, wait); } /* Modem status lines are not supported. */ @@ -4265,7 +4246,7 @@ static void gsm_dtr_rts(struct tty_port *port, bool active) modem_tx &= ~(TIOCM_DTR | TIOCM_RTS); if (modem_tx != dlci->modem_tx) { dlci->modem_tx = modem_tx; - gsm_modem_update(dlci, 0); + gsm_modem_update(dlci, 0, true); } } @@ -4471,7 +4452,7 @@ static int gsmtty_tiocmset(struct tty_struct *tty, if (modem_tx != dlci->modem_tx) { dlci->modem_tx = modem_tx; - return gsm_modem_update(dlci, 0); + return gsm_modem_update(dlci, 0, true); } return 0; } @@ -4553,7 +4534,7 @@ static void gsmtty_throttle(struct tty_struct *tty) dlci->modem_tx &= ~TIOCM_RTS; dlci->throttled = true; /* Send an MSC with RTS cleared */ - gsm_modem_update(dlci, 0); + gsm_modem_update(dlci, 0, true); } static void gsmtty_unthrottle(struct tty_struct *tty) @@ -4565,7 +4546,7 @@ static void gsmtty_unthrottle(struct tty_struct *tty) dlci->modem_tx |= TIOCM_RTS; dlci->throttled = false; /* Send an MSC with RTS set */ - gsm_modem_update(dlci, 0); + gsm_modem_update(dlci, 0, true); } static int gsmtty_break_ctl(struct tty_struct *tty, int state) @@ -4583,7 +4564,7 @@ static int gsmtty_break_ctl(struct tty_struct *tty, int state) if (encode > 0x0F) encode = 0x0F; /* Best effort */ } - return gsm_modem_update(dlci, encode); + return gsm_modem_update(dlci, encode, true); } static void gsmtty_cleanup(struct tty_struct *tty) diff --git a/drivers/tty/serial/8250/8250_bcm7271.c b/drivers/tty/serial/8250/8250_bcm7271.c index cb7d594d263e..5f1140f05150 100644 --- a/drivers/tty/serial/8250/8250_bcm7271.c +++ b/drivers/tty/serial/8250/8250_bcm7271.c @@ -1121,8 +1121,8 @@ static void brcmuart_remove(struct platform_device *pdev) struct brcmuart_priv *priv = platform_get_drvdata(pdev); debugfs_remove_recursive(priv->debugfs_dir); - hrtimer_cancel(&priv->hrt); serial8250_unregister_port(priv->line); + hrtimer_cancel(&priv->hrt); brcmuart_free_bufs(&pdev->dev, priv); if (priv->dma_enabled) brcmuart_arbitration(priv, 0); diff --git a/drivers/tty/serial/8250/8250_omap.c b/drivers/tty/serial/8250/8250_omap.c index 3c7775df27ef..ceecb39fb82d 100644 --- a/drivers/tty/serial/8250/8250_omap.c +++ b/drivers/tty/serial/8250/8250_omap.c @@ -711,12 +711,6 @@ static int omap_8250_startup(struct uart_port *port) struct uart_8250_dma *dma = &priv->omap8250_dma; int ret; - if (priv->wakeirq) { - ret = dev_pm_set_dedicated_wake_irq(port->dev, priv->wakeirq); - if (ret) - return ret; - } - #ifdef CONFIG_PM up->capabilities |= UART_CAP_RPM; #endif @@ -787,7 +781,6 @@ static void omap_8250_shutdown(struct uart_port *port) } disable_irq_nosync(port->irq); - dev_pm_clear_wake_irq(port->dev); serial8250_release_dma(up); up->dma = NULL; @@ -1588,6 +1581,12 @@ static int omap8250_probe(struct platform_device *pdev) priv->wakeirq = irq_of_parse_and_map(np, 1); + if (priv->wakeirq) { + ret = dev_pm_set_dedicated_wake_irq(&pdev->dev, priv->wakeirq); + if (ret) + goto err; + } + ret = serial8250_register_8250_port(&up); if (ret < 0) { dev_err(&pdev->dev, "unable to register 8250 port\n"); @@ -1603,6 +1602,7 @@ static int omap8250_probe(struct platform_device *pdev) return 0; err: + dev_pm_clear_wake_irq(&pdev->dev); pm_runtime_dont_use_autosuspend(&pdev->dev); pm_runtime_put_sync(&pdev->dev); flush_work(&priv->qos_work); @@ -1625,6 +1625,7 @@ static void omap8250_remove(struct platform_device *pdev) omap_8250_shutdown(&up->port); serial8250_unregister_port(priv->line); priv->line = -ENODEV; + dev_pm_clear_wake_irq(&pdev->dev); pm_runtime_dont_use_autosuspend(&pdev->dev); pm_runtime_put_sync(&pdev->dev); flush_work(&priv->qos_work); diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c index 38fa45e74a37..1acb73e45f8a 100644 --- a/drivers/tty/serial/8250/8250_port.c +++ b/drivers/tty/serial/8250/8250_port.c @@ -3597,7 +3597,7 @@ int serial8250_console_setup(struct uart_port *port, char *options, bool probe) up->console_line_ended = true; up->console_msr_work_allow = true; - init_irq_work(&up->console_msr_work, console_msr_handler); + up->console_msr_work = IRQ_WORK_INIT_LAZY(console_msr_handler); if (options) uart_parse_options(options, &baud, &parity, &bits, &flow); diff --git a/drivers/tty/serial/kgdboc.c b/drivers/tty/serial/kgdboc.c index 09648d643897..64e3f8cf2f55 100644 --- a/drivers/tty/serial/kgdboc.c +++ b/drivers/tty/serial/kgdboc.c @@ -188,6 +188,11 @@ static void cleanup_kgdboc(void) kgdboc_unregister_kbd(); kgdb_unregister_io_module(&kgdboc_io_ops); + + if (kgdb_tty_driver) { + tty_driver_kref_put(kgdb_tty_driver); + kgdb_tty_driver = NULL; + } } static int configure_kgdboc(void) @@ -254,6 +259,10 @@ static int configure_kgdboc(void) noconfig: kgdboc_unregister_kbd(); + if (kgdb_tty_driver) { + tty_driver_kref_put(kgdb_tty_driver); + kgdb_tty_driver = NULL; + } configured = 0; return err; diff --git a/drivers/tty/serial/ma35d1_serial.c b/drivers/tty/serial/ma35d1_serial.c index 920fe7ff5083..0809655f4395 100644 --- a/drivers/tty/serial/ma35d1_serial.c +++ b/drivers/tty/serial/ma35d1_serial.c @@ -728,7 +728,7 @@ static int ma35d1serial_probe(struct platform_device *pdev) ret = clk_prepare_enable(up->clk); if (ret) - goto err_iounmap; + goto err_put_clk; if (up->port.line != 0) up->port.uartclk = clk_get_rate(up->clk); @@ -755,6 +755,9 @@ static int ma35d1serial_probe(struct platform_device *pdev) err_clk_disable: clk_disable_unprepare(up->clk); +err_put_clk: + clk_put(up->clk); + err_iounmap: iounmap(up->port.membase); return ret; @@ -770,6 +773,7 @@ static void ma35d1serial_remove(struct platform_device *dev) uart_remove_one_port(&ma35d1serial_reg, port); clk_disable_unprepare(up->clk); + clk_put(up->clk); } static int ma35d1serial_suspend(struct platform_device *dev, pm_message_t state) diff --git a/drivers/tty/serial/max3100.c b/drivers/tty/serial/max3100.c index a5a5cd68f696..f858a6669011 100644 --- a/drivers/tty/serial/max3100.c +++ b/drivers/tty/serial/max3100.c @@ -535,11 +535,11 @@ static void max3100_shutdown(struct uart_port *port) timer_delete_sync(&s->timer); if (s->workqueue) { + free_irq(port->irq, s); + cancel_work_sync(&s->work); destroy_workqueue(s->workqueue); s->workqueue = NULL; } - if (port->irq) - free_irq(port->irq, s); /* set shutdown mode to save power */ max3100_sr(s, MAX3100_WC | MAX3100_SHDN, &rx); @@ -753,6 +753,15 @@ static void max3100_remove(struct spi_device *spi) if (max3100s[i] == s) { dev_dbg(&spi->dev, "%s: removing port %d\n", __func__, i); uart_remove_one_port(&max3100_uart_driver, &max3100s[i]->port); + + s->force_end_work = 1; + timer_shutdown_sync(&s->timer); + if (s->workqueue) { + free_irq(s->port.irq, s); + cancel_work_sync(&s->work); + destroy_workqueue(s->workqueue); + s->workqueue = NULL; + } kfree(max3100s[i]); max3100s[i] = NULL; break; diff --git a/drivers/tty/serial/mpc52xx_uart.c b/drivers/tty/serial/mpc52xx_uart.c index b566206f42a2..472ec39caf27 100644 --- a/drivers/tty/serial/mpc52xx_uart.c +++ b/drivers/tty/serial/mpc52xx_uart.c @@ -634,6 +634,9 @@ static void __exit mpc512x_psc_fifoc_uninit(void) } } +static struct clk *psc_mclk_clk[MPC52xx_PSC_MAXNUM]; +static struct clk *psc_ipg_clk[MPC52xx_PSC_MAXNUM]; + /* 512x specific interrupt handler. The caller holds the port lock */ static irqreturn_t mpc512x_psc_handle_irq(struct uart_port *port) { @@ -654,9 +657,6 @@ static irqreturn_t mpc512x_psc_handle_irq(struct uart_port *port) return IRQ_NONE; } -static struct clk *psc_mclk_clk[MPC52xx_PSC_MAXNUM]; -static struct clk *psc_ipg_clk[MPC52xx_PSC_MAXNUM]; - /* called from within the .request_port() callback (allocation) */ static int mpc512x_psc_alloc_clock(struct uart_port *port) { diff --git a/drivers/tty/serial/qcom_geni_serial.c b/drivers/tty/serial/qcom_geni_serial.c index 3633723acef8..a851b8f0ef04 100644 --- a/drivers/tty/serial/qcom_geni_serial.c +++ b/drivers/tty/serial/qcom_geni_serial.c @@ -171,7 +171,6 @@ static void qcom_geni_serial_cancel_tx_cmd(struct uart_port *uport); static int qcom_geni_serial_port_setup(struct uart_port *uport); static void qcom_geni_serial_start_tx_fifo(struct uart_port *uport); static void qcom_geni_serial_resume_tx(struct uart_port *uport); -static void qcom_geni_serial_poll_rx_fifo_locked(struct uart_port *uport); static inline struct qcom_geni_serial_port *to_dev_port(struct uart_port *uport) { @@ -467,6 +466,102 @@ static int qcom_geni_serial_poll_init(struct uart_port *uport) #endif #ifdef CONFIG_SERIAL_QCOM_GENI_CONSOLE +static void handle_rx_console(struct uart_port *uport, u32 bytes, bool drop) +{ + u32 i; + unsigned char buf[sizeof(u32)]; + struct tty_port *tport; + struct qcom_geni_serial_port *port = to_dev_port(uport); + + tport = &uport->state->port; + for (i = 0; i < bytes; ) { + int c; + int chunk = min_t(int, bytes - i, BYTES_PER_FIFO_WORD); + + ioread32_rep(uport->membase + SE_GENI_RX_FIFOn, buf, 1); + i += chunk; + if (drop) + continue; + + for (c = 0; c < chunk; c++) { + int sysrq; + + uport->icount.rx++; + if (port->brk && buf[c] == 0) { + port->brk = false; + if (uart_handle_break(uport)) + continue; + } + + sysrq = uart_prepare_sysrq_char(uport, buf[c]); + + if (!sysrq) + tty_insert_flip_char(tport, buf[c], TTY_NORMAL); + } + } + if (!drop) + tty_flip_buffer_push(tport); +} +#else +static void handle_rx_console(struct uart_port *uport, u32 bytes, bool drop) +{ + +} +#endif + +static void qcom_geni_serial_handle_rx_fifo(struct uart_port *uport, bool drop) +{ + u32 status; + u32 word_cnt; + u32 last_word_byte_cnt; + u32 last_word_partial; + u32 total_bytes; + + status = readl(uport->membase + SE_GENI_RX_FIFO_STATUS); + word_cnt = status & RX_FIFO_WC_MSK; + last_word_partial = status & RX_LAST; + last_word_byte_cnt = (status & RX_LAST_BYTE_VALID_MSK) >> + RX_LAST_BYTE_VALID_SHFT; + + if (!word_cnt) + return; + total_bytes = BYTES_PER_FIFO_WORD * (word_cnt - 1); + if (last_word_partial && last_word_byte_cnt) + total_bytes += last_word_byte_cnt; + else + total_bytes += BYTES_PER_FIFO_WORD; + handle_rx_console(uport, total_bytes, drop); +} + +#ifdef CONFIG_SERIAL_QCOM_GENI_CONSOLE +/* Caller holds the UART port lock. */ +static void qcom_geni_serial_poll_rx_fifo_locked(struct uart_port *uport) +{ + struct qcom_geni_serial_port *port = to_dev_port(uport); + struct tty_port *tport = &uport->state->port; + u32 s_irq_status; + bool drop_rx = false; + + s_irq_status = readl(uport->membase + SE_GENI_S_IRQ_STATUS); + writel(s_irq_status, uport->membase + SE_GENI_S_IRQ_CLEAR); + + if (s_irq_status & S_RX_FIFO_WR_ERR_EN) { + uport->icount.overrun++; + tty_insert_flip_char(tport, 0, TTY_OVERRUN); + } + + if (s_irq_status & (S_GP_IRQ_0_EN | S_GP_IRQ_1_EN)) { + if (s_irq_status & S_GP_IRQ_0_EN) + uport->icount.parity++; + drop_rx = true; + } else if (s_irq_status & (S_GP_IRQ_2_EN | S_GP_IRQ_3_EN)) { + uport->icount.brk++; + port->brk = true; + } + + qcom_geni_serial_handle_rx_fifo(uport, drop_rx); +} + static void qcom_geni_serial_drain_fifo(struct uart_port *uport) { struct qcom_geni_serial_port *port = to_dev_port(uport); @@ -676,47 +771,6 @@ static void qcom_geni_serial_console_device_unlock(struct console *co, #endif } -static void handle_rx_console(struct uart_port *uport, u32 bytes, bool drop) -{ - u32 i; - unsigned char buf[sizeof(u32)]; - struct tty_port *tport; - struct qcom_geni_serial_port *port = to_dev_port(uport); - - tport = &uport->state->port; - for (i = 0; i < bytes; ) { - int c; - int chunk = min_t(int, bytes - i, BYTES_PER_FIFO_WORD); - - ioread32_rep(uport->membase + SE_GENI_RX_FIFOn, buf, 1); - i += chunk; - if (drop) - continue; - - for (c = 0; c < chunk; c++) { - int sysrq; - - uport->icount.rx++; - if (port->brk && buf[c] == 0) { - port->brk = false; - if (uart_handle_break(uport)) - continue; - } - - sysrq = uart_prepare_sysrq_char(uport, buf[c]); - - if (!sysrq) - tty_insert_flip_char(tport, buf[c], TTY_NORMAL); - } - } - if (!drop) - tty_flip_buffer_push(tport); -} -#else -static void handle_rx_console(struct uart_port *uport, u32 bytes, bool drop) -{ - -} #endif /* CONFIG_SERIAL_QCOM_GENI_CONSOLE */ static void handle_rx_uart(struct uart_port *uport, u32 bytes) @@ -890,58 +944,6 @@ static void qcom_geni_serial_cancel_tx_cmd(struct uart_port *uport) port->tx_queued = 0; } -static void qcom_geni_serial_handle_rx_fifo(struct uart_port *uport, bool drop) -{ - u32 status; - u32 word_cnt; - u32 last_word_byte_cnt; - u32 last_word_partial; - u32 total_bytes; - - status = readl(uport->membase + SE_GENI_RX_FIFO_STATUS); - word_cnt = status & RX_FIFO_WC_MSK; - last_word_partial = status & RX_LAST; - last_word_byte_cnt = (status & RX_LAST_BYTE_VALID_MSK) >> - RX_LAST_BYTE_VALID_SHFT; - - if (!word_cnt) - return; - total_bytes = BYTES_PER_FIFO_WORD * (word_cnt - 1); - if (last_word_partial && last_word_byte_cnt) - total_bytes += last_word_byte_cnt; - else - total_bytes += BYTES_PER_FIFO_WORD; - handle_rx_console(uport, total_bytes, drop); -} - -/* Caller holds the UART port lock. */ -static void qcom_geni_serial_poll_rx_fifo_locked(struct uart_port *uport) -{ - struct qcom_geni_serial_port *port = to_dev_port(uport); - struct tty_port *tport = &uport->state->port; - u32 s_irq_status; - bool drop_rx = false; - - s_irq_status = readl(uport->membase + SE_GENI_S_IRQ_STATUS); - writel(s_irq_status, uport->membase + SE_GENI_S_IRQ_CLEAR); - - if (s_irq_status & S_RX_FIFO_WR_ERR_EN) { - uport->icount.overrun++; - tty_insert_flip_char(tport, 0, TTY_OVERRUN); - } - - if (s_irq_status & (S_GP_IRQ_0_EN | S_GP_IRQ_1_EN)) { - if (s_irq_status & S_GP_IRQ_0_EN) - uport->icount.parity++; - drop_rx = true; - } else if (s_irq_status & (S_GP_IRQ_2_EN | S_GP_IRQ_3_EN)) { - uport->icount.brk++; - port->brk = true; - } - - qcom_geni_serial_handle_rx_fifo(uport, drop_rx); -} - static void qcom_geni_serial_stop_rx_fifo(struct uart_port *uport) { u32 irq_en; @@ -1482,9 +1484,8 @@ static int geni_serial_set_rate(struct geni_se *se, unsigned long baud) * Bump up BW vote on CPU and CORE path as driver supports FIFO mode * only. */ - avg_bw_core = (baud > 115200) ? Bps_to_icc(CORE_2X_50_MHZ) - : GENI_DEFAULT_BW; - port->se.icc_paths[GENI_TO_CORE].avg_bw = avg_bw_core; + avg_bw_core = baud > 115200 ? CORE_2X_50_MHZ : CORE_2X_19_2_MHZ; + port->se.icc_paths[GENI_TO_CORE].avg_bw = Bps_to_icc(avg_bw_core); port->se.icc_paths[CPU_TO_GENI].avg_bw = Bps_to_icc(baud); geni_icc_set_bw(&port->se); @@ -1625,16 +1626,37 @@ static int qcom_geni_console_setup(struct console *co, char *options) if (unlikely(!uport->membase)) return -ENXIO; + ret = pm_runtime_resume_and_get(uport->dev); + if (ret < 0) + return ret; + if (!port->setup) { ret = qcom_geni_serial_port_setup(uport); - if (ret) + if (ret) { + pm_runtime_put_sync(uport->dev); return ret; + } } if (options) uart_parse_options(options, &baud, &parity, &bits, &flow); - return uart_set_options(uport, co, baud, parity, bits, flow); + ret = uart_set_options(uport, co, baud, parity, bits, flow); + if (ret) + pm_runtime_put_sync(uport->dev); + + return ret; +} + +static int qcom_geni_console_exit(struct console *co) +{ + struct qcom_geni_serial_port *port; + + port = get_port_from_line(co->index, true, NULL); + if (IS_ERR(port)) + return PTR_ERR(port); + + return pm_runtime_put_sync(port->uport.dev); } static void qcom_geni_serial_earlycon_write(struct console *con, @@ -1751,6 +1773,7 @@ static struct console cons_ops = { .device_unlock = qcom_geni_serial_console_device_unlock, .device = uart_console_device, .setup = qcom_geni_console_setup, + .exit = qcom_geni_console_exit, .flags = CON_PRINTBUFFER | CON_NBCON, .index = -1, .data = &qcom_geni_console_driver, @@ -2099,9 +2122,11 @@ static int qcom_geni_serial_resume(struct device *dev) struct uart_port *uport = &port->uport; struct qcom_geni_private_data *private_data = uport->private_data; - ret = pm_runtime_force_resume(dev); - if (ret) - return ret; + if (console_suspend_enabled || !uart_console(uport)) { + ret = pm_runtime_force_resume(dev); + if (ret) + return ret; + } ret = uart_resume_port(private_data->drv, uport); if (uart_console(uport)) { diff --git a/drivers/tty/serial/sc16is7xx.c b/drivers/tty/serial/sc16is7xx.c index 7107a0fb1e7b..296079c16fdb 100644 --- a/drivers/tty/serial/sc16is7xx.c +++ b/drivers/tty/serial/sc16is7xx.c @@ -216,6 +216,8 @@ #define SC16IS7XX_TLR_TX_TRIGGER(words) ((((words) / 4) & 0x0f) << 0) #define SC16IS7XX_TLR_RX_TRIGGER(words) ((((words) / 4) & 0x0f) << 4) +#define SC16IS7XX_TX_TRIGGER_LEVEL 32 + /* IOControl register bits (Only 75x/76x) */ #define SC16IS7XX_IOCONTROL_LATCH_BIT BIT(0) /* Enable input latching */ #define SC16IS7XX_IOCONTROL_MODEM_A_BIT BIT(1) /* Enable GPIO[7:4] as modem A pins */ @@ -647,12 +649,26 @@ static void sc16is7xx_handle_rx(struct uart_port *port, unsigned int rxlen, tty_flip_buffer_push(&port->state->port); } +static unsigned int sc16is7xx_txlvl(struct uart_port *port) +{ + unsigned int txlvl; + + txlvl = sc16is7xx_port_read(port, SC16IS7XX_TXLVL_REG); + if (txlvl > SC16IS7XX_FIFO_SIZE) { + dev_err_ratelimited(port->dev, + "chip reports %u free bytes in TX FIFO, but it only has %u\n", + txlvl, SC16IS7XX_FIFO_SIZE); + return 0; + } + + return txlvl; +} + static void sc16is7xx_handle_tx(struct uart_port *port) { struct tty_port *tport = &port->state->port; unsigned long flags; unsigned int txlen; - unsigned char *tail; if (unlikely(port->x_char)) { sc16is7xx_port_write(port, SC16IS7XX_THR_REG, port->x_char); @@ -669,17 +685,28 @@ static void sc16is7xx_handle_tx(struct uart_port *port) } /* Limit to space available in TX FIFO */ - txlen = sc16is7xx_port_read(port, SC16IS7XX_TXLVL_REG); - if (txlen > SC16IS7XX_FIFO_SIZE) { - dev_err_ratelimited(port->dev, - "chip reports %d free bytes in TX fifo, but it only has %d", - txlen, SC16IS7XX_FIFO_SIZE); - txlen = 0; - } + txlen = sc16is7xx_txlvl(port); - txlen = kfifo_out_linear_ptr(&tport->xmit_fifo, &tail, txlen); - sc16is7xx_fifo_write(port, tail, txlen); - uart_xmit_advance(port, txlen); + /* Handle circular buffer wrap-around by sending multiple segments */ + while (txlen > 0 && !kfifo_is_empty(&tport->xmit_fifo)) { + unsigned char *tail; + unsigned int to_send; + + to_send = kfifo_out_linear_ptr(&tport->xmit_fifo, &tail, txlen); + if (!to_send) + break; + + sc16is7xx_fifo_write(port, tail, to_send); + uart_xmit_advance(port, to_send); + + if (kfifo_is_empty(&tport->xmit_fifo)) + break; + + /* Refill below the trigger to enable the next THRI crossing. */ + txlen = sc16is7xx_txlvl(port); + if (txlen < SC16IS7XX_TX_TRIGGER_LEVEL) + break; + } uart_port_lock_irqsave(port, &flags); if (kfifo_len(&tport->xmit_fifo) < WAKEUP_CHARS) @@ -1130,6 +1157,10 @@ static int sc16is7xx_startup(struct uart_port *port) SC16IS7XX_TCR_RX_RESUME(24) | SC16IS7XX_TCR_RX_HALT(48)); + /* Sync hardware and software TX trigger levels */ + sc16is7xx_port_write(port, SC16IS7XX_TLR_REG, + SC16IS7XX_TLR_TX_TRIGGER(SC16IS7XX_TX_TRIGGER_LEVEL)); + /* Disable TCR/TLR access */ sc16is7xx_port_update(port, SC16IS7XX_MCR_REG, SC16IS7XX_MCR_TCRTLR_BIT, 0); diff --git a/drivers/tty/serial/serial-tegra.c b/drivers/tty/serial/serial-tegra.c index 8004fc00fb9c..b0fc4ae3686f 100644 --- a/drivers/tty/serial/serial-tegra.c +++ b/drivers/tty/serial/serial-tegra.c @@ -306,11 +306,19 @@ static void tegra_uart_fifo_reset(struct tegra_uart_port *tup, u8 fcr_bits) { unsigned long fcr = tup->fcr_shadow; unsigned int lsr, tmout = 10000; + bool clear_tx = !!(fcr_bits & UART_FCR_CLEAR_XMIT); if (tup->rts_active) set_rts(tup, false); - if (tup->cdata->allow_txfifo_reset_fifo_mode) { + /* + * Leaving FIFO mode below is a workaround for a Tegra30 + * restriction on clearing the Tx FIFO while FIFO mode is + * enabled. It empties both FIFOs, so applying it to an + * Rx-only reset would destroy an in-flight transmission. + * Only take that path when the caller asked for CLEAR_XMIT. + */ + if (tup->cdata->allow_txfifo_reset_fifo_mode || !clear_tx) { fcr |= fcr_bits & (UART_FCR_CLEAR_RCVR | UART_FCR_CLEAR_XMIT); tegra_uart_write(tup, fcr, UART_FCR); } else { @@ -335,9 +343,15 @@ static void tegra_uart_fifo_reset(struct tegra_uart_port *tup, u8 fcr_bits) */ tegra_uart_wait_cycle_time(tup, 32); + /* + * Only wait for the transmitter to drain when the Tx FIFO was + * part of the reset. For an Rx-only reset it is left intact, + * and waiting for TEMT here would spin for a full frame time. + */ do { lsr = tegra_uart_read(tup, UART_LSR); - if ((lsr & UART_LSR_TEMT) && !(lsr & UART_LSR_DR)) + if ((!clear_tx || (lsr & UART_LSR_TEMT)) && + !(lsr & UART_LSR_DR)) break; udelay(1); } while (--tmout); diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c index 95774b0f1484..6332ed545c89 100644 --- a/drivers/tty/serial/serial_core.c +++ b/drivers/tty/serial/serial_core.c @@ -896,7 +896,7 @@ static int uart_set_info(struct tty_struct *tty, struct tty_port *port, upf_t old_flags, new_flags; int retval; - if (!uport) + if (!uport || tty_io_error(tty)) return -EIO; new_port = new_info->port; @@ -1119,7 +1119,7 @@ static int uart_break_ctl(struct tty_struct *tty, int break_state) guard(mutex)(&port->mutex); uport = uart_port_check(state); - if (!uport) + if (!uport || tty_io_error(tty)) return -EIO; if (uport->type != PORT_UNKNOWN && uport->ops->break_ctl) @@ -1144,7 +1144,7 @@ static int uart_do_autoconfig(struct tty_struct *tty, struct uart_state *state) */ scoped_cond_guard(mutex_intr, return -ERESTARTSYS, &port->mutex) { uport = uart_port_check(state); - if (!uport) + if (!uport || tty_io_error(tty)) return -EIO; if (tty_port_users(port) != 1) @@ -1199,7 +1199,7 @@ static void uart_enable_ms(struct uart_port *uport) * FIXME: This wants extracting into a common all driver implementation * of TIOCMWAIT using tty_port. */ -static int uart_wait_modem_status(struct uart_state *state, unsigned long arg) +static int uart_wait_modem_status(struct tty_struct *tty, struct uart_state *state, unsigned long arg) { struct uart_port *uport; struct tty_port *port = &state->port; @@ -1213,18 +1213,29 @@ static int uart_wait_modem_status(struct uart_state *state, unsigned long arg) uport = uart_port_ref(state); if (!uport) return -EIO; - scoped_guard(uart_port_lock_irq, uport) { - memcpy(&cprev, &uport->icount, sizeof(struct uart_icount)); - uart_enable_ms(uport); + + mutex_lock(&port->mutex); + if (tty_io_error(tty)) { + mutex_unlock(&port->mutex); + ret = -EIO; + goto out_deref; } + uart_port_lock_irq(uport); + memcpy(&cprev, &uport->icount, sizeof(struct uart_icount)); + uart_enable_ms(uport); + uart_port_unlock_irq(uport); + + mutex_unlock(&port->mutex); + add_wait_queue(&port->delta_msr_wait, &wait); for (;;) { - scoped_guard(uart_port_lock_irq, uport) - memcpy(&cnow, &uport->icount, sizeof(struct uart_icount)); - set_current_state(TASK_INTERRUPTIBLE); + uart_port_lock_irq(uport); + memcpy(&cnow, &uport->icount, sizeof(struct uart_icount)); + uart_port_unlock_irq(uport); + if (((arg & TIOCM_RNG) && (cnow.rng != cprev.rng)) || ((arg & TIOCM_DSR) && (cnow.dsr != cprev.dsr)) || ((arg & TIOCM_CD) && (cnow.dcd != cprev.dcd)) || @@ -1233,6 +1244,11 @@ static int uart_wait_modem_status(struct uart_state *state, unsigned long arg) break; } + if (tty_io_error(tty)) { + ret = -EIO; + break; + } + schedule(); /* see if a signal did it */ @@ -1245,6 +1261,7 @@ static int uart_wait_modem_status(struct uart_state *state, unsigned long arg) } __set_current_state(TASK_RUNNING); remove_wait_queue(&port->delta_msr_wait, &wait); +out_deref: uart_port_deref(uport); return ret; @@ -1567,7 +1584,7 @@ uart_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned long arg) /* This should only be used when the hardware is present. */ if (cmd == TIOCMIWAIT) - return uart_wait_modem_status(state, arg); + return uart_wait_modem_status(tty, state, arg); /* rs485_config requires more locking than others */ if (cmd == TIOCSRS485) @@ -1623,14 +1640,13 @@ static void uart_set_ldisc(struct tty_struct *tty) { struct uart_state *state = tty->driver_data; struct uart_port *uport; - struct tty_port *port = &state->port; - - if (!tty_port_initialized(port)) - return; guard(mutex)(&state->port.mutex); uport = uart_port_check(state); - if (uport && uport->ops->set_ldisc) + if (!uport || tty_io_error(tty)) + return; + + if (uport->ops->set_ldisc) uport->ops->set_ldisc(uport, &tty->termios); } @@ -1646,7 +1662,7 @@ static void uart_set_termios(struct tty_struct *tty, guard(mutex)(&state->port.mutex); uport = uart_port_check(state); - if (!uport) + if (!uport || tty_io_error(tty)) return; /* @@ -1798,7 +1814,14 @@ static void uart_wait_until_sent(struct tty_struct *tty, int timeout) * 'timeout' / 'expire' give us the maximum amount of time * we wait. */ - while (!port->ops->tx_empty(port)) { + for (;;) { + mutex_lock(&state->port.mutex); + if (tty_io_error(tty) || port->ops->tx_empty(port)) { + mutex_unlock(&state->port.mutex); + break; + } + mutex_unlock(&state->port.mutex); + msleep_interruptible(jiffies_to_msecs(char_time)); if (signal_pending(current)) break; @@ -3358,6 +3381,7 @@ int serial_core_register_port(struct uart_driver *drv, struct uart_port *port) err_unregister_port_dev: serial_base_port_device_remove(port->port_dev); + port->port_dev = NULL; err_unregister_ctrl_dev: serial_base_ctrl_device_remove(new_ctrl_dev); @@ -3372,12 +3396,24 @@ int serial_core_register_port(struct uart_driver *drv, struct uart_port *port) void serial_core_unregister_port(struct uart_driver *drv, struct uart_port *port) { struct device *phys_dev = port->dev; - struct serial_port_device *port_dev = port->port_dev; - struct serial_ctrl_device *ctrl_dev = serial_core_get_ctrl_dev(port_dev); + struct serial_port_device *port_dev; + struct serial_ctrl_device *ctrl_dev; int ctrl_id = port->ctrl_id; guard(mutex)(&port_mutex); + /* + * A NULL port device means there is no registered port device to + * remove: serial_core_remove_one_port() clears port_dev on + * teardown, and it is never set if registration failed before + * serial_core_port_device_add(). + */ + port_dev = port->port_dev; + if (!port_dev) + return; + + ctrl_dev = serial_core_get_ctrl_dev(port_dev); + port->flags |= UPF_DEAD; serial_core_remove_one_port(drv, port); diff --git a/drivers/tty/serial/vt8500_serial.c b/drivers/tty/serial/vt8500_serial.c index 78a1c1eea11b..fc6fdc751b16 100644 --- a/drivers/tty/serial/vt8500_serial.c +++ b/drivers/tty/serial/vt8500_serial.c @@ -656,6 +656,7 @@ static int vt8500_serial_probe(struct platform_device *pdev) ret = clk_prepare_enable(vt8500_port->clk); if (ret) { dev_err(&pdev->dev, "failed to enable clock\n"); + clk_put(vt8500_port->clk); return ret; } diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c index 48569035da56..1a6c8a1bc764 100644 --- a/drivers/tty/tty_io.c +++ b/drivers/tty/tty_io.c @@ -620,6 +620,8 @@ static void __tty_hangup(struct tty_struct *tty, int exit_session) tty_ldisc_hangup(tty, cons_filp != NULL); + wake_up_interruptible(&tty->break_wait); + spin_lock_irq(&tty->ctrl.lock); clear_bit(TTY_THROTTLED, &tty->flags); clear_bit(TTY_DO_WRITE_WAKEUP, &tty->flags); @@ -2431,6 +2433,7 @@ static int tiocgetd(struct tty_struct *tty, int __user *p) * send_break - performed time break * @tty: device to break on * @duration: timeout in mS + * @file: file object * * Perform a timed break on hardware that lacks its own driver level timed * break functionality. @@ -2438,8 +2441,9 @@ static int tiocgetd(struct tty_struct *tty, int __user *p) * Locking: * @tty->atomic_write_lock serializes */ -static int send_break(struct tty_struct *tty, unsigned int duration) +static int send_break(struct file *file, struct tty_struct *tty, unsigned int duration) { + long timeout; int retval; if (tty->ops->break_ctl == NULL) @@ -2453,13 +2457,26 @@ static int send_break(struct tty_struct *tty, unsigned int duration) return -EINTR; retval = tty->ops->break_ctl(tty, -1); - if (!retval) { - msleep_interruptible(duration); - retval = tty->ops->break_ctl(tty, 0); - } else if (retval == -EOPNOTSUPP) { - /* some drivers can tell only dynamically */ - retval = 0; + if (retval) { + if (retval == -EOPNOTSUPP) { + /* some drivers can tell only dynamically */ + retval = 0; + } + goto out_unlock; } + + timeout = msecs_to_jiffies(duration); + timeout = wait_event_interruptible_timeout(tty->break_wait, + tty_hung_up_p(file), + timeout); + /* return early on hangup only */ + if (timeout > 0) { + retval = -EIO; + goto out_unlock; + } + + retval = tty->ops->break_ctl(tty, 0); +out_unlock: tty_write_unlock(tty); if (signal_pending(current)) @@ -2727,10 +2744,10 @@ long tty_ioctl(struct file *file, unsigned int cmd, unsigned long arg) * This is used by the tcdrain() termios function. */ if (!arg) - return send_break(tty, 250); + return send_break(file, tty, 250); return 0; case TCSBRKP: /* support for POSIX tcsendbreak() */ - return send_break(tty, arg ? arg*100 : 250); + return send_break(file, tty, arg ? arg * 100 : 250); case TIOCMGET: return tty_tiocmget(tty, p); @@ -3090,6 +3107,7 @@ struct tty_struct *alloc_tty_struct(struct tty_driver *driver, int idx) init_ldsem(&tty->ldisc_sem); init_waitqueue_head(&tty->write_wait); init_waitqueue_head(&tty->read_wait); + init_waitqueue_head(&tty->break_wait); INIT_WORK(&tty->hangup_work, do_tty_hangup); mutex_init(&tty->atomic_write_lock); spin_lock_init(&tty->ctrl.lock); @@ -3238,14 +3256,16 @@ struct device *tty_register_device_attr(struct tty_driver *driver, goto err_put; if (!(driver->flags & TTY_DRIVER_DYNAMIC_ALLOC)) { - /* - * Free any saved termios data so that the termios state is - * reset when reusing a minor number. - */ - tp = driver->termios[index]; - if (tp) { - driver->termios[index] = NULL; - kfree(tp); + if (driver->flags & TTY_DRIVER_RESET_SAVED_TERMIOS) { + /* + * Free any saved termios data so that the termios + * state is reset when reusing a minor number. + */ + tp = driver->termios[index]; + if (tp) { + driver->termios[index] = NULL; + kfree(tp); + } } retval = tty_cdev_add(driver, devt, index, 1); diff --git a/drivers/tty/tty_port.c b/drivers/tty/tty_port.c index 54359310e293..b22ff9eb7778 100644 --- a/drivers/tty/tty_port.c +++ b/drivers/tty/tty_port.c @@ -340,19 +340,9 @@ void tty_port_tty_set(struct tty_port *port, struct tty_struct *tty) } EXPORT_SYMBOL(tty_port_tty_set); -/** - * tty_port_shutdown - internal helper to shutdown the device - * @port: tty port to be shut down - * @tty: the associated tty - * - * It is used by tty_port_hangup() and tty_port_close(). Its task is to - * shutdown the device if it was initialized (note consoles remain - * functioning). It lowers DTR/RTS (if @tty has HUPCL set) and invokes - * @port->ops->shutdown(). - */ -static void tty_port_shutdown(struct tty_port *port, struct tty_struct *tty) +static void tty_port_shutdown_locked(struct tty_port *port, struct tty_struct *tty) { - guard(mutex)(&port->mutex); + lockdep_assert_held(&port->mutex); if (port->console) return; @@ -372,6 +362,23 @@ static void tty_port_shutdown(struct tty_port *port, struct tty_struct *tty) port->ops->shutdown(port); } +/** + * tty_port_shutdown - internal helper to shutdown the device + * @port: tty port to be shut down + * @tty: the associated tty + * + * It is used by tty_port_hangup() and tty_port_close(). Its task is to + * shutdown the device if it was initialized (note consoles remain + * functioning). It lowers DTR/RTS (if @tty has HUPCL set) and invokes + * @port->ops->shutdown(). + */ +static void tty_port_shutdown(struct tty_port *port, struct tty_struct *tty) +{ + guard(mutex)(&port->mutex); + + tty_port_shutdown_locked(port, tty); +} + /** * tty_port_hangup - hangup helper * @port: tty port @@ -385,36 +392,58 @@ void tty_port_hangup(struct tty_port *port) { struct tty_struct *tty; - scoped_guard(spinlock_irqsave, &port->lock) { - port->count = 0; - tty = port->tty; - if (tty) - set_bit(TTY_IO_ERROR, &tty->flags); - port->tty = NULL; - } + scoped_guard(mutex, &port->mutex) { + scoped_guard(spinlock_irqsave, &port->lock) { + port->count = 0; + tty = port->tty; + if (tty) + set_bit(TTY_IO_ERROR, &tty->flags); + port->tty = NULL; + } - tty_port_set_active(port, false); - tty_port_shutdown(port, tty); + tty_port_set_active(port, false); + tty_port_shutdown_locked(port, tty); + } tty_kref_put(tty); wake_up_interruptible(&port->open_wait); wake_up_interruptible(&port->delta_msr_wait); } EXPORT_SYMBOL(tty_port_hangup); -void __tty_port_tty_hangup(struct tty_port *port, bool check_clocal, bool async) +/** + * tty_port_tty_hangup - helper to hang up a tty asynchronously + * @port: tty port + * @check_clocal: hang only ttys with %CLOCAL unset? + */ +void tty_port_tty_hangup(struct tty_port *port, bool check_clocal) { scoped_guard(tty_port_tty, port) { struct tty_struct *tty = scoped_tty(); - if (!check_clocal || !C_CLOCAL(tty)) { - if (async) - tty_hangup(tty); - else - tty_vhangup(tty); - } + if (!check_clocal || !C_CLOCAL(tty)) + tty_hangup(tty); } } -EXPORT_SYMBOL_GPL(__tty_port_tty_hangup); +EXPORT_SYMBOL_GPL(tty_port_tty_hangup); + +/** + * tty_port_tty_vhangup - helper to hang up a tty synchronously + * @port: tty port + */ +void tty_port_tty_vhangup(struct tty_port *port) +{ + struct tty_struct *tty; + + mutex_lock(&port->mutex); + tty = tty_port_tty_get(port); + mutex_unlock(&port->mutex); + + if (tty) { + tty_vhangup(tty); + tty_kref_put(tty); + } +} +EXPORT_SYMBOL_GPL(tty_port_tty_vhangup); /** * tty_port_tty_wakeup - helper to wake up a tty @@ -767,8 +796,10 @@ int tty_port_open(struct tty_port *port, struct tty_struct *tty, clear_bit(TTY_IO_ERROR, &tty->flags); if (port->ops->activate) { int retval = port->ops->activate(port, tty); - if (retval) + if (retval) { + set_bit(TTY_IO_ERROR, &tty->flags); return retval; + } } tty_port_set_initialized(port, true); } diff --git a/drivers/tty/vcc.c b/drivers/tty/vcc.c index 27a55465bf5e..9adb533f7034 100644 --- a/drivers/tty/vcc.c +++ b/drivers/tty/vcc.c @@ -492,7 +492,7 @@ static ssize_t domain_show(struct device *dev, static int vcc_send_ctl(struct vcc_port *port, int ctl) { - struct vio_vcc pkt; + struct vio_vcc pkt = {}; int rv; pkt.tag.type = VIO_TYPE_CTRL; @@ -986,7 +986,8 @@ static void vcc_cleanup(struct tty_struct *tty) port = vcc_get(tty->index, true); if (port) { - port->tty = NULL; + scoped_guard(spinlock_irqsave, &port->lock) + port->tty = NULL; if (port->removed) { vcc_table_remove(tty->index); diff --git a/drivers/tty/vt/selection.c b/drivers/tty/vt/selection.c index 13f4e48b4142..f1a3bc5b5fad 100644 --- a/drivers/tty/vt/selection.c +++ b/drivers/tty/vt/selection.c @@ -434,8 +434,8 @@ int paste_selection(struct tty_struct *tty) bps = NULL; } - count = vc_sel.buf_len - pasted; - if (count) { + if (vc_sel.buf_len > pasted) { + count = vc_sel.buf_len - pasted; pasted += tty_ldisc_receive_buf(ld, vc_sel.buffer + pasted, NULL, count); if (vc_sel.buf_len > pasted) diff --git a/drivers/tty/vt/vc_screen.c b/drivers/tty/vt/vc_screen.c index bf1502fd5bd4..79453abcf4d9 100644 --- a/drivers/tty/vt/vc_screen.c +++ b/drivers/tty/vt/vc_screen.c @@ -631,6 +631,18 @@ vcs_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) ret = copy_from_user(con_buf, buf, this_round); console_lock(); + /* The vc might have been freed or vcs_size might have changed + * while we slept to grab the user buffer; recheck here, before + * if (ret), so every break path below passes a fresh vc to the + * post-loop vcs_scr_updated(). Return data written so far. + */ + vc = vcs_vc(inode, &viewed); + if (!vc) { + if (written) + break; + return -ENXIO; + } + if (ret) { this_round -= ret; if (!this_round) { @@ -642,17 +654,6 @@ vcs_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) return -EFAULT; } } - - /* The vc might have been freed or vcs_size might have changed - * while we slept to grab the user buffer, so recheck. - * Return data written up to now on failure. - */ - vc = vcs_vc(inode, &viewed); - if (!vc) { - if (written) - break; - return -ENXIO; - } size = vcs_size(vc, attr, false); if (size < 0) { if (written) diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index 57edf37495a8..49a140b49eff 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -1134,6 +1134,8 @@ int vc_allocate(unsigned int currcons) /* return 0 on success */ return 0; err_free: visual_deinit(vc); + if (*vc->uni_pagedict_loc) + con_free_unimap(vc); kfree(vc); vc_cons[currcons].d = NULL; return err; @@ -2860,7 +2862,8 @@ static void do_con_trol(struct tty_struct *tty, struct vc_data *vc, u8 c) csi_J(vc, CSI_J_VISIBLE); vc->vc_video_erase_char = (vc->vc_video_erase_char & 0xff00) | ' '; - do_update_region(vc, vc->vc_origin, vc->vc_screenbuf_size / 2); + if (con_should_update(vc)) + do_update_region(vc, vc->vc_origin, vc->vc_screenbuf_size / 2); } return; case ESsetG0: /* ESC ( */ diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 2ba244cf40ac..54f4e7d7de02 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -6817,11 +6817,17 @@ static void ufshcd_err_handling_prepare(struct ufs_hba *hba) } /* Wait for ongoing ufshcd_queuecommand() calls to finish. */ blk_mq_quiesce_tagset(&hba->host->tag_set); + /* + * Internal commands are submitted on the pseudo SCSI device. Let them + * through so that the error handler can recover the link. + */ + blk_mq_unquiesce_queue(hba->host->pseudo_sdev->request_queue); cancel_work_sync(&hba->eeh_work); } static void ufshcd_err_handling_unprepare(struct ufs_hba *hba) { + blk_mq_quiesce_queue_nowait(hba->host->pseudo_sdev->request_queue); blk_mq_unquiesce_tagset(&hba->host->tag_set); ufshcd_release(hba); if (ufshcd_is_clkscaling_supported(hba)) diff --git a/drivers/ufs/host/ufshcd-pltfrm.c b/drivers/ufs/host/ufshcd-pltfrm.c index 5ac7afe75934..169dbc1a75fe 100644 --- a/drivers/ufs/host/ufshcd-pltfrm.c +++ b/drivers/ufs/host/ufshcd-pltfrm.c @@ -206,7 +206,11 @@ static void ufshcd_init_lanes_per_dir(struct ufs_hba *hba) dev_dbg(hba->dev, "%s: failed to read lanes-per-direction, ret=%d\n", __func__, ret); - hba->lanes_per_direction = UFSHCD_DEFAULT_LANES_PER_DIRECTION; + /* Old R-Car S4 DTBs lack "lanes-per-direction = <1>" */ + if (of_device_is_compatible(dev->of_node, "renesas,r8a779f0-ufs")) + hba->lanes_per_direction = 1; + else + hba->lanes_per_direction = UFSHCD_DEFAULT_LANES_PER_DIRECTION; } } diff --git a/drivers/usb/cdns3/cdns3-gadget.c b/drivers/usb/cdns3/cdns3-gadget.c index 42311c1bfada..0d272e9c556c 100644 --- a/drivers/usb/cdns3/cdns3-gadget.c +++ b/drivers/usb/cdns3/cdns3-gadget.c @@ -3269,6 +3269,10 @@ static void cdns3_gadget_exit(struct cdns *cdns) usb_del_gadget(&priv_dev->gadget); devm_free_irq(cdns->dev, cdns->dev_irq, priv_dev); + /* The works can still be queued until the IRQ is freed. */ + cancel_work_sync(&priv_dev->pending_status_wq); + cancel_work_sync(&priv_dev->aligned_buf_wq); + cdns3_free_all_eps(priv_dev); while (!list_empty(&priv_dev->aligned_buf_list)) { diff --git a/drivers/usb/cdns3/cdns3-pci-wrap.c b/drivers/usb/cdns3/cdns3-pci-wrap.c index eb5760f75b9d..fd06a3d8e638 100644 --- a/drivers/usb/cdns3/cdns3-pci-wrap.c +++ b/drivers/usb/cdns3/cdns3-pci-wrap.c @@ -96,6 +96,11 @@ static int cdns3_pci_probe(struct pci_dev *pdev, if (pci_is_enabled(func)) { wrap = pci_get_drvdata(func); + if (!wrap) { + dev_err(&pdev->dev, + "second function not initialized, retrying\n"); + return -EPROBE_DEFER; + } } else { wrap = kzalloc_obj(*wrap); if (!wrap) diff --git a/drivers/usb/chipidea/ci_hdrc_tegra.c b/drivers/usb/chipidea/ci_hdrc_tegra.c index 372788f0f970..26225b034d3b 100644 --- a/drivers/usb/chipidea/ci_hdrc_tegra.c +++ b/drivers/usb/chipidea/ci_hdrc_tegra.c @@ -306,8 +306,10 @@ static int tegra_usb_probe(struct platform_device *pdev) pm_runtime_enable(&pdev->dev); err = pm_runtime_resume_and_get(&pdev->dev); - if (err) + if (err) { + pm_runtime_disable(&pdev->dev); return err; + } if (device_property_present(&pdev->dev, "nvidia,needs-double-reset")) usb->needs_double_reset = true; diff --git a/drivers/usb/class/cdc-acm.c b/drivers/usb/class/cdc-acm.c index 7bc5329fa3ed..0d3f815b2f1e 100644 --- a/drivers/usb/class/cdc-acm.c +++ b/drivers/usb/class/cdc-acm.c @@ -331,7 +331,6 @@ static void acm_process_notification(struct acm *acm, unsigned char *buf) spin_lock_irqsave(&acm->read_lock, flags); acm->ctrlin = newctrl; - acm->oldcount = acm->iocount; if (difference & USB_CDC_SERIAL_STATE_DSR) acm->iocount.dsr++; @@ -788,6 +787,9 @@ static void acm_port_shutdown(struct tty_port *port) usb_autopm_put_interface_async(acm->control); } + if (acm->disconnected) + return; + acm_unpoison_urbs(acm); if (acm->quirks & ALWAYS_POLL_CTRL) { @@ -1027,11 +1029,16 @@ static int wait_serial_change(struct acm *acm, unsigned long arg) DECLARE_WAITQUEUE(wait, current); struct async_icount old, new; - do { + spin_lock_irq(&acm->read_lock); + old = acm->iocount; + spin_unlock_irq(&acm->read_lock); + + add_wait_queue(&acm->wioctl, &wait); + for (;;) { + set_current_state(TASK_INTERRUPTIBLE); + spin_lock_irq(&acm->read_lock); - old = acm->oldcount; new = acm->iocount; - acm->oldcount = new; spin_unlock_irq(&acm->read_lock); if ((arg & TIOCM_DSR) && @@ -1044,22 +1051,20 @@ static int wait_serial_change(struct acm *acm, unsigned long arg) old.rng != new.rng) break; - add_wait_queue(&acm->wioctl, &wait); - set_current_state(TASK_INTERRUPTIBLE); - schedule(); - remove_wait_queue(&acm->wioctl, &wait); if (acm->disconnected) { - if (arg & TIOCM_CD) - break; - else - rv = -ENODEV; - } else { - if (signal_pending(current)) - rv = -ERESTARTSYS; + rv = -ENODEV; + break; } - } while (!rv); - + schedule(); + + if (signal_pending(current)) { + rv = -ERESTARTSYS; + break; + } + } + __set_current_state(TASK_RUNNING); + remove_wait_queue(&acm->wioctl, &wait); return rv; } @@ -2140,7 +2145,7 @@ static int __init acm_init(void) { int retval; acm_tty_driver = tty_alloc_driver(ACM_TTY_MINORS, TTY_DRIVER_REAL_RAW | - TTY_DRIVER_DYNAMIC_DEV); + TTY_DRIVER_DYNAMIC_DEV | TTY_DRIVER_RESET_SAVED_TERMIOS); if (IS_ERR(acm_tty_driver)) return PTR_ERR(acm_tty_driver); acm_tty_driver->driver_name = "acm", diff --git a/drivers/usb/class/cdc-acm.h b/drivers/usb/class/cdc-acm.h index 01f448a783c0..d245e60eb9f5 100644 --- a/drivers/usb/class/cdc-acm.h +++ b/drivers/usb/class/cdc-acm.h @@ -90,7 +90,6 @@ struct acm { unsigned int ctrlin; /* input control lines (DCD, DSR, RI, break, overruns) */ unsigned int ctrlout; /* output control lines (DTR, RTS) */ struct async_icount iocount; /* counters for control line changes */ - struct async_icount oldcount; /* for comparison of counter */ wait_queue_head_t wioctl; /* for ioctl */ unsigned int writesize; /* max packet size for the output bulk endpoint */ unsigned int readsize,ctrlsize; /* buffer sizes for freeing */ diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c index 3345b3298daf..b446905b48be 100644 --- a/drivers/usb/core/hub.c +++ b/drivers/usb/core/hub.c @@ -73,8 +73,11 @@ /* * Give SS hubs 200ms time after wake to train downstream links before * assuming no port activity and allowing hub to runtime suspend back. + * Root hubs have no upstream hub whose wake propagation needs to be + * accounted for, so they need less time, use 120ms for them. */ #define USB_SS_PORT_U0_WAKE_TIME 200 /* ms */ +#define USB_SS_RH_PORT_U0_WAKE_TIME 120 /* ms */ /* Protect struct usb_device->state and ->children members * Note: Both are also protected by ->dev.sem, except that ->state can @@ -1358,7 +1361,9 @@ static void hub_activate(struct usb_hub *hub, enum hub_activation_type type) queue_delayed_work(system_power_efficient_wq, &hub->post_resume_work, - msecs_to_jiffies(USB_SS_PORT_U0_WAKE_TIME)); + msecs_to_jiffies(hdev->parent ? + USB_SS_PORT_U0_WAKE_TIME : + USB_SS_RH_PORT_U0_WAKE_TIME)); return; } diff --git a/drivers/usb/core/message.c b/drivers/usb/core/message.c index 75e2bfd744a9..0cd6dd2b6334 100644 --- a/drivers/usb/core/message.c +++ b/drivers/usb/core/message.c @@ -1337,7 +1337,8 @@ static void remove_intf_ep_devs(struct usb_interface *intf) * * Disables the endpoint for URB submission and nukes all pending URBs. * If @reset_hardware is set then also deallocates hcd/hardware state - * for the endpoint. + * for the endpoint (clearing both ep_in and ep_out pointers for + * bidirectional non-ep0 control endpoints). */ void usb_disable_endpoint(struct usb_device *dev, unsigned int epaddr, bool reset_hardware) @@ -1358,6 +1359,11 @@ void usb_disable_endpoint(struct usb_device *dev, unsigned int epaddr, dev->ep_in[epnum] = NULL; } if (ep) { + if (reset_hardware && epnum != 0 && + usb_endpoint_xfer_control(&ep->desc)) { + dev->ep_out[epnum] = NULL; + dev->ep_in[epnum] = NULL; + } ep->enabled = 0; usb_hcd_flush_endpoint(dev, ep); if (reset_hardware) diff --git a/drivers/usb/dwc2/hcd.c b/drivers/usb/dwc2/hcd.c index 2414291aa908..cd0dc876b421 100644 --- a/drivers/usb/dwc2/hcd.c +++ b/drivers/usb/dwc2/hcd.c @@ -5082,14 +5082,13 @@ static void dwc2_hcd_free(struct dwc2_hsotg *hsotg) } cancel_work_sync(&hsotg->phy_reset_work); - - timer_delete(&hsotg->wkp_timer); } static void dwc2_hcd_release(struct dwc2_hsotg *hsotg) { /* Turn off all host-specific interrupts */ dwc2_disable_host_interrupts(hsotg); + timer_shutdown_sync(&hsotg->wkp_timer); dwc2_hcd_free(hsotg); } diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h index 608daeb7ef10..f3dabb37f806 100644 --- a/drivers/usb/dwc3/core.h +++ b/drivers/usb/dwc3/core.h @@ -49,6 +49,7 @@ #define DWC3_ENDPOINTS_NUM 32 #define DWC3_XHCI_RESOURCES_NUM 2 #define DWC3_ISOC_MAX_RETRIES 5 +#define DWC3_ERR_RECOVERY_MAX 3 #define DWC3_SCRATCHBUF_SIZE 4096 /* each buffer is assumed to be 4KiB */ #define DWC3_EVENT_BUFFERS_SIZE 4096 @@ -841,6 +842,12 @@ enum dwc3_link_state { DWC3_LINK_STATE_MASK = 0x0f, }; +enum dwc3_err_state { + DWC3_ERR_NONE = 0, + DWC3_ERR_RECOVERY, + DWC3_ERR_UNRECOVERABLE, +}; + /* TRB Length, PCM and Status */ #define DWC3_TRB_SIZE_MASK (0x00ffffff) #define DWC3_TRB_SIZE_LENGTH(n) ((n) & DWC3_TRB_SIZE_MASK) @@ -1004,6 +1011,7 @@ struct dwc3_glue_ops { /** * struct dwc3 - representation of our controller * @drd_work: workqueue used for role swapping + * @err_recovery_work: workqueue used for controller error recovery * @ep0_trb: trb which is used for the ctrl_req * @bounce: address of bounce buffer * @setup_buf: used while precessing STD USB requests @@ -1013,6 +1021,7 @@ struct dwc3_glue_ops { * @ep0_in_setup: one control transfer is completed and enter setup phase * @lock: for synchronizing * @mutex: for mode switching + * @connect_mutex: for the pull-up and err_recovery_work * @dev: pointer to our struct device * @sysdev: pointer to the DMA-capable device * @xhci: pointer to our xHCI child @@ -1079,6 +1088,9 @@ struct dwc3_glue_ops { * @ep0_next_event: hold the next expected event * @ep0state: state of endpoint zero * @link_state: link state + * @err_state: current error recovery state. + * @err_recovery_count: number of consecutive error recovery attempts until + * confirmed healthy and reset to 0 on reset event. * @speed: device speed (super, high, full, low) * @hwparams: copy of hwparams registers * @regset: debugfs pointer to regdump file @@ -1189,6 +1201,7 @@ struct dwc3_glue_ops { */ struct dwc3 { struct work_struct drd_work; + struct work_struct err_recovery_work; struct dwc3_trb *ep0_trb; void *bounce; u8 *setup_buf; @@ -1203,6 +1216,9 @@ struct dwc3 { /* mode switching lock */ struct mutex mutex; + /* serializes pull-up run/stop vs error recovery */ + struct mutex connect_mutex; + struct device *dev; struct device *sysdev; @@ -1332,6 +1348,9 @@ struct dwc3 { enum dwc3_ep0_next ep0_next_event; enum dwc3_ep0_state ep0state; enum dwc3_link_state link_state; + enum dwc3_err_state err_state; + + u32 err_recovery_count; u16 u2sel; u16 u2pel; diff --git a/drivers/usb/dwc3/dwc3-am62.c b/drivers/usb/dwc3/dwc3-am62.c index 632634d6e81e..d03b3950ce04 100644 --- a/drivers/usb/dwc3/dwc3-am62.c +++ b/drivers/usb/dwc3/dwc3-am62.c @@ -299,6 +299,7 @@ static int dwc3_ti_probe(struct platform_device *pdev) err_pm_disable: clk_disable_unprepare(am62->usb2_refclk); + pm_runtime_put_noidle(dev); pm_runtime_disable(dev); pm_runtime_set_suspended(dev); return ret; diff --git a/drivers/usb/dwc3/ep0.c b/drivers/usb/dwc3/ep0.c index 310b5ffb236a..46a34c4a046b 100644 --- a/drivers/usb/dwc3/ep0.c +++ b/drivers/usb/dwc3/ep0.c @@ -200,6 +200,11 @@ int dwc3_gadget_ep0_queue(struct usb_ep *ep, struct usb_request *request, int ret; spin_lock_irqsave(&dwc->lock, flags); + if (dwc->err_state != DWC3_ERR_NONE) { + ret = -ESHUTDOWN; + goto out; + } + if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) { dev_err(dwc->dev, "%s: can't queue to disabled endpoint\n", dep->name); @@ -271,6 +276,10 @@ int dwc3_gadget_ep0_set_halt(struct usb_ep *ep, int value) int ret; spin_lock_irqsave(&dwc->lock, flags); + if (dwc->err_state != DWC3_ERR_NONE) { + spin_unlock_irqrestore(&dwc->lock, flags); + return -ESHUTDOWN; + } ret = __dwc3_gadget_ep0_set_halt(ep, value); spin_unlock_irqrestore(&dwc->lock, flags); diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index f245e66cd13d..ee837235630a 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -1149,6 +1149,10 @@ static int dwc3_gadget_ep_enable(struct usb_ep *ep, return 0; spin_lock_irqsave(&dwc->lock, flags); + if (dwc->err_state != DWC3_ERR_NONE) { + spin_unlock_irqrestore(&dwc->lock, flags); + return -ESHUTDOWN; + } ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT); spin_unlock_irqrestore(&dwc->lock, flags); @@ -2060,6 +2064,10 @@ static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request, int ret; spin_lock_irqsave(&dwc->lock, flags); + if (dwc->err_state != DWC3_ERR_NONE) { + spin_unlock_irqrestore(&dwc->lock, flags); + return -ESHUTDOWN; + } ret = __dwc3_gadget_ep_queue(dep, req); spin_unlock_irqrestore(&dwc->lock, flags); @@ -2292,6 +2300,10 @@ static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value) int ret; spin_lock_irqsave(&dwc->lock, flags); + if (dwc->err_state != DWC3_ERR_NONE) { + spin_unlock_irqrestore(&dwc->lock, flags); + return -ESHUTDOWN; + } ret = __dwc3_gadget_ep_set_halt(dep, value, false); spin_unlock_irqrestore(&dwc->lock, flags); @@ -2306,6 +2318,10 @@ static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep) int ret; spin_lock_irqsave(&dwc->lock, flags); + if (dwc->err_state != DWC3_ERR_NONE) { + spin_unlock_irqrestore(&dwc->lock, flags); + return -ESHUTDOWN; + } dep->flags |= DWC3_EP_WEDGE; if (dep->number == 0 || dep->number == 1) @@ -2437,6 +2453,10 @@ static int dwc3_gadget_wakeup(struct usb_gadget *g) } spin_lock_irqsave(&dwc->lock, flags); + if (dwc->err_state != DWC3_ERR_NONE) { + spin_unlock_irqrestore(&dwc->lock, flags); + return -ESHUTDOWN; + } if (!dwc->gadget->wakeup_armed) { dev_err(dwc->dev, "not armed for remote wakeup\n"); spin_unlock_irqrestore(&dwc->lock, flags); @@ -2464,6 +2484,10 @@ static int dwc3_gadget_func_wakeup(struct usb_gadget *g, int intf_id) } spin_lock_irqsave(&dwc->lock, flags); + if (dwc->err_state != DWC3_ERR_NONE) { + spin_unlock_irqrestore(&dwc->lock, flags); + return -ESHUTDOWN; + } /* * If the link is in U3, signal for remote wakeup and wait for the * link to transition to U0 before sending device notification. @@ -2833,10 +2857,13 @@ static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on) synchronize_irq(dwc->irq_gadget); + /* Serialize against dwc3_err_recovery_work() */ + mutex_lock(&dwc->connect_mutex); if (!is_on) ret = dwc3_gadget_soft_disconnect(dwc); else ret = dwc3_gadget_soft_connect(dwc); + mutex_unlock(&dwc->connect_mutex); pm_runtime_put(dwc->dev); @@ -4153,6 +4180,10 @@ static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc) dwc->suspended = false; + /* The controller is recovered. */ + if (dwc->err_state == DWC3_ERR_NONE) + dwc->err_recovery_count = 0; + /* * Ideally, dwc3_reset_gadget() would trigger the function * drivers to stop any active transfers through ep disable. @@ -4640,6 +4671,12 @@ static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt) return ret; } +static void dwc3_schedule_err_recovery(struct dwc3 *dwc) +{ + dwc->err_state = DWC3_ERR_RECOVERY; + schedule_work(&dwc->err_recovery_work); +} + static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt) { struct dwc3 *dwc = evt->dwc; @@ -4673,9 +4710,21 @@ static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt) return IRQ_NONE; if (count > evt->length) { - dev_err_ratelimited(dwc->dev, "invalid count(%u) > evt->length(%u)\n", + dev_err(dwc->dev, "invalid count(%u) > evt->length(%u)\n", count, evt->length); - return IRQ_NONE; + /* + * This is a fatal error - the driver and controller are out of + * sync on which event has been consumed. Reinitializing the + * controller is required to recover. Write the bogus count back + * to GEVNTCOUNT to clear the IRQ source, consistent with the + * stale event clearing in dwc3_event_buffers_setup(), then + * schedule error recovery. + */ + spin_lock(&dwc->lock); + dwc3_schedule_err_recovery(dwc); + spin_unlock(&dwc->lock); + dwc3_writel(dwc, DWC3_GEVNTCOUNT(0), count); + return IRQ_HANDLED; } evt->count = count; @@ -4735,6 +4784,57 @@ static void dwc_gadget_release(struct device *dev) kfree(gadget); } +static void dwc3_err_recovery_work(struct work_struct *work) +{ + struct dwc3 *dwc = container_of(work, struct dwc3, err_recovery_work); + unsigned long flags; + int ret; + + /* serializes against dwc3_gadget_pullup() */ + mutex_lock(&dwc->connect_mutex); + + ret = dwc3_gadget_soft_disconnect(dwc); + + dwc3_disconnect_gadget_sleepable(dwc); + + if (ret) + goto err_unrecoverable; + + if (dwc->softconnect) { + u32 count; + /* + * Wait irq to finish before soft_connect resets evt->lpos and + * the event buffer registers to avoid racing with + * dwc3_process_event_buf(). + */ + synchronize_irq(dwc->irq_gadget); + + spin_lock_irqsave(&dwc->lock, flags); + count = ++dwc->err_recovery_count; + spin_unlock_irqrestore(&dwc->lock, flags); + + if (count > DWC3_ERR_RECOVERY_MAX) + goto err_unrecoverable; + + ret = dwc3_gadget_soft_connect(dwc); + if (ret) + goto err_unrecoverable; + } + mutex_unlock(&dwc->connect_mutex); + + spin_lock_irqsave(&dwc->lock, flags); + dwc->err_state = DWC3_ERR_NONE; + spin_unlock_irqrestore(&dwc->lock, flags); + return; + +err_unrecoverable: + dev_err(dwc->dev, "Unable to recover the controller\n"); + mutex_unlock(&dwc->connect_mutex); + spin_lock_irqsave(&dwc->lock, flags); + dwc->err_state = DWC3_ERR_UNRECOVERABLE; + spin_unlock_irqrestore(&dwc->lock, flags); +} + /** * dwc3_gadget_init - initializes gadget related registers * @dwc: pointer to our controller context structure @@ -4778,6 +4878,8 @@ int dwc3_gadget_init(struct dwc3 *dwc) } init_completion(&dwc->ep0_in_setup); + INIT_WORK(&dwc->err_recovery_work, dwc3_err_recovery_work); + mutex_init(&dwc->connect_mutex); dwc->gadget = kzalloc_obj(struct usb_gadget); if (!dwc->gadget) { ret = -ENOMEM; @@ -4874,6 +4976,8 @@ void dwc3_gadget_exit(struct dwc3 *dwc) if (!dwc->gadget) return; + cancel_work_sync(&dwc->err_recovery_work); + mutex_destroy(&dwc->connect_mutex); dwc3_enable_susphy(dwc, true); usb_del_gadget(dwc->gadget); dwc3_gadget_free_endpoints(dwc); diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c index 43962e05eacf..c64a268e98a4 100644 --- a/drivers/usb/gadget/function/f_fs.c +++ b/drivers/usb/gadget/function/f_fs.c @@ -1877,8 +1877,9 @@ static long ffs_epfile_ioctl(struct file *file, unsigned code, break; case FUNCTIONFS_ENDPOINT_DESC: { + const struct usb_endpoint_descriptor *desc; + struct usb_endpoint_descriptor desc1; int desc_idx; - struct usb_endpoint_descriptor desc1, *desc; switch (epfile->ffs->gadget->speed) { case USB_SPEED_SUPER: @@ -1892,7 +1893,17 @@ static long ffs_epfile_ioctl(struct file *file, unsigned code, desc_idx = 0; } - desc = epfile->ep->descs[desc_idx]; + do { + desc = epfile->ep->descs[desc_idx]; + } while (!desc && --desc_idx >= 0); + + if (!desc) + desc = epfile->ep->ep->desc; + if (!desc) { + ret = -EINVAL; + break; + } + memcpy(&desc1, desc, desc->bLength); spin_unlock_irq(&epfile->ffs->eps_lock); diff --git a/drivers/usb/gadget/function/f_midi2.c b/drivers/usb/gadget/function/f_midi2.c index 5b8b18281989..d693de427641 100644 --- a/drivers/usb/gadget/function/f_midi2.c +++ b/drivers/usb/gadget/function/f_midi2.c @@ -1633,8 +1633,8 @@ struct f_midi2_usb_config { /* MIDI 1.0 jacks */ unsigned char jack_in, jack_out, jack_id; - struct usb_midi_in_jack_descriptor jack_ins[MAX_CABLES]; - struct usb_midi_out_jack_descriptor_1 jack_outs[MAX_CABLES]; + struct usb_midi_in_jack_descriptor jack_ins[MAX_CABLES * 2]; + struct usb_midi_out_jack_descriptor_1 jack_outs[MAX_CABLES * 2]; }; static int append_config(struct f_midi2_usb_config *config, void *d) diff --git a/drivers/usb/gadget/function/f_uac1_legacy.c b/drivers/usb/gadget/function/f_uac1_legacy.c index 3f52099a4fdd..7c8d60c7e87d 100644 --- a/drivers/usb/gadget/function/f_uac1_legacy.c +++ b/drivers/usb/gadget/function/f_uac1_legacy.c @@ -797,6 +797,9 @@ f_audio_bind(struct usb_configuration *c, struct usb_function *f) static int generic_set_cmd(struct usb_audio_control *con, u8 cmd, int value) { + if (cmd >= ARRAY_SIZE(con->data)) + return -EINVAL; + con->data[cmd] = value; return 0; @@ -804,6 +807,9 @@ static int generic_set_cmd(struct usb_audio_control *con, u8 cmd, int value) static int generic_get_cmd(struct usb_audio_control *con, u8 cmd) { + if (cmd >= ARRAY_SIZE(con->data)) + return -EINVAL; + return con->data[cmd]; } diff --git a/drivers/usb/gadget/udc/aspeed-vhub/core.c b/drivers/usb/gadget/udc/aspeed-vhub/core.c index 4a8b9ff8368f..069673f0d359 100644 --- a/drivers/usb/gadget/udc/aspeed-vhub/core.c +++ b/drivers/usb/gadget/udc/aspeed-vhub/core.c @@ -267,6 +267,9 @@ static void ast_vhub_remove(struct platform_device *pdev) for (i = 0; i < vhub->max_ports; i++) ast_vhub_del_dev(&vhub->ports[i].dev); + /* Final drain; the worker takes vhub->lock, so stay outside of it */ + cancel_work_sync(&vhub->wake_work); + spin_lock_irqsave(&vhub->lock, flags); /* Mask & ack all interrupts */ @@ -328,6 +331,7 @@ static int ast_vhub_probe(struct platform_device *pdev) return -ENOMEM; spin_lock_init(&vhub->lock); + INIT_WORK(&vhub->wake_work, ast_vhub_wake_work); vhub->pdev = pdev; vhub->port_irq_mask = GENMASK(VHUB_IRQ_DEV1_BIT + vhub->max_ports - 1, VHUB_IRQ_DEV1_BIT); diff --git a/drivers/usb/gadget/udc/aspeed-vhub/dev.c b/drivers/usb/gadget/udc/aspeed-vhub/dev.c index 8b9449d16324..4b389de0d935 100644 --- a/drivers/usb/gadget/udc/aspeed-vhub/dev.c +++ b/drivers/usb/gadget/udc/aspeed-vhub/dev.c @@ -280,7 +280,7 @@ static int ast_vhub_udc_wakeup(struct usb_gadget* gadget) int rc = -EINVAL; spin_lock_irqsave(&d->vhub->lock, flags); - if (!d->wakeup_en) + if (!d->wakeup_en || !d->registered) goto err; DDBG(d, "Device initiated wakeup\n"); diff --git a/drivers/usb/gadget/udc/aspeed-vhub/hub.c b/drivers/usb/gadget/udc/aspeed-vhub/hub.c index 02fe1a08d575..d0345f31037a 100644 --- a/drivers/usb/gadget/udc/aspeed-vhub/hub.c +++ b/drivers/usb/gadget/udc/aspeed-vhub/hub.c @@ -558,7 +558,7 @@ void ast_vhub_device_connect(struct ast_vhub *vhub, ast_vhub_send_host_wakeup(vhub); } -static void ast_vhub_wake_work(struct work_struct *work) +void ast_vhub_wake_work(struct work_struct *work) { struct ast_vhub *vhub = container_of(work, struct ast_vhub, @@ -588,6 +588,8 @@ static void ast_vhub_wake_work(struct work_struct *work) void ast_vhub_hub_wake_all(struct ast_vhub *vhub) { + lockdep_assert_held(&vhub->lock); + /* * A device is trying to wake the world, because this * can recurse into the device, we break the call chain @@ -1076,7 +1078,6 @@ static int ast_vhub_init_desc(struct ast_vhub *vhub) int ast_vhub_init_hub(struct ast_vhub *vhub) { vhub->speed = USB_SPEED_UNKNOWN; - INIT_WORK(&vhub->wake_work, ast_vhub_wake_work); return ast_vhub_init_desc(vhub); } diff --git a/drivers/usb/gadget/udc/aspeed-vhub/vhub.h b/drivers/usb/gadget/udc/aspeed-vhub/vhub.h index aca2050e2db0..96f1c3709042 100644 --- a/drivers/usb/gadget/udc/aspeed-vhub/vhub.h +++ b/drivers/usb/gadget/udc/aspeed-vhub/vhub.h @@ -547,6 +547,7 @@ void ast_vhub_hub_suspend(struct ast_vhub *vhub); void ast_vhub_hub_resume(struct ast_vhub *vhub); void ast_vhub_hub_reset(struct ast_vhub *vhub); void ast_vhub_hub_wake_all(struct ast_vhub *vhub); +void ast_vhub_wake_work(struct work_struct *work); /* dev.c */ int ast_vhub_init_dev(struct ast_vhub *vhub, unsigned int idx); diff --git a/drivers/usb/gadget/udc/dummy_hcd.c b/drivers/usb/gadget/udc/dummy_hcd.c index c0e40fa6dde5..0d30dd67b9ca 100644 --- a/drivers/usb/gadget/udc/dummy_hcd.c +++ b/drivers/usb/gadget/udc/dummy_hcd.c @@ -343,9 +343,11 @@ static void dummy_giveback(struct dummy *dum, struct usb_ep *_ep, { bool fifo = req == &dum->fifo_req; + ++dum->callback_usage; spin_unlock(&dum->lock); usb_gadget_giveback_request(_ep, &req->req); spin_lock(&dum->lock); + --dum->callback_usage; if (fifo) dum->fifo_req_busy = 0; } @@ -759,11 +761,13 @@ static int dummy_queue(struct usb_ep *_ep, struct usb_request *_req, req->req.complete = fifo_complete; list_add_tail(&req->queue, &ep->queue); + ++dum->callback_usage; spin_unlock(&dum->lock); _req->actual = _req->length; _req->status = 0; usb_gadget_giveback_request(_ep, _req); spin_lock(&dum->lock); + --dum->callback_usage; } else list_add_tail(&req->queue, &ep->queue); spin_unlock_irqrestore(&dum->lock, flags); diff --git a/drivers/usb/host/octeon-hcd.c b/drivers/usb/host/octeon-hcd.c index 34ce771fa0f5..256e459ff6d8 100644 --- a/drivers/usb/host/octeon-hcd.c +++ b/drivers/usb/host/octeon-hcd.c @@ -378,8 +378,22 @@ struct octeon_hcd { struct cvmx_usb_transaction *active_split; struct cvmx_usb_tx_fifo periodic; struct cvmx_usb_tx_fifo nonperiodic; + struct hrtimer sof_timer; }; +static int cvmx_usb_poll(struct octeon_hcd *usb); + +static enum hrtimer_restart octeon_usb_sof_timer(struct hrtimer *t) +{ + struct octeon_hcd *usb = container_of(t, struct octeon_hcd, sof_timer); + unsigned long flags; + + spin_lock_irqsave(&usb->lock, flags); + cvmx_usb_poll(usb); + spin_unlock_irqrestore(&usb->lock, flags); + return HRTIMER_NORESTART; +} + /* * This macro logically sets a single field in a CSR. It does the sequence * read, modify, and write @@ -578,7 +592,7 @@ static int cvmx_wait_tx_rx(struct octeon_hcd *usb, int fflsh_type) int result; u64 address = CVMX_USBCX_GRSTCTL(usb->index); u64 done = cvmx_get_cycle() + 100 * - (u64)octeon_get_clock_rate / 1000000; + (u64)octeon_get_clock_rate() / 1000000; union cvmx_usbcx_grstctl c; while (1) { @@ -853,6 +867,14 @@ static int cvmx_usb_initialize(struct device *dev, * USBC_GAHBCFG[PTXFEMPLVL] * Global interrupt mask, USBC_GAHBCFG[GLBLINTRMSK] = 1 */ + usbcx_gahbcfg.u32 = cvmx_usb_read_csr32(usb, + CVMX_USBCX_GHWCFG3(usb->index)); + if (usbcx_gahbcfg.u32 == 0xffffffff || usbcx_gahbcfg.u32 == 0) { + dev_err(dev, "USB core is not responding (GHWCFG3=0x%08x)\n", + usbcx_gahbcfg.u32); + return -ENODEV; + } + usbcx_gahbcfg.u32 = 0; usbcx_gahbcfg.s.dmaen = !(usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA); @@ -1900,6 +1922,7 @@ static void cvmx_usb_schedule(struct octeon_hcd *usb, int is_sof) int channel; struct cvmx_usb_pipe *pipe; int need_sof; + u64 min_due; enum cvmx_usb_transfer ttype; if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) { @@ -1940,15 +1963,33 @@ static void cvmx_usb_schedule(struct octeon_hcd *usb, int is_sof) * future that might need to be scheduled */ need_sof = 0; + min_due = ~0ull; for (ttype = CVMX_USB_TRANSFER_CONTROL; ttype <= CVMX_USB_TRANSFER_INTERRUPT; ttype++) { list_for_each_entry(pipe, &usb->active_pipes[ttype], node) { - if (pipe->next_tx_frame > usb->frame_number) { - need_sof = 1; - break; - } + if (pipe->next_tx_frame > usb->frame_number && + pipe->next_tx_frame < min_due) + min_due = pipe->next_tx_frame; } } + if (min_due != ~0ull) { + u64 delta = min_due - usb->frame_number; + + /* + * frame_number is resynced from HFNUM on every poll, so a + * deadline that is many frames away does not need an + * interrupt on every SOF to count them down - sleep on the + * timer instead and keep SOF interrupts for deadlines within + * a few frames. Stay well below the 16383-frame wrap of + * HFNUM. One (micro)frame is 125us in high-speed mode. + */ + if (delta <= 4 || delta > 8000) + need_sof = 1; + else + hrtimer_start(&usb->sof_timer, + ns_to_ktime((delta - 2) * 125000), + HRTIMER_MODE_REL); + } USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index), cvmx_usbcx_gintmsk, sofmsk, need_sof); } @@ -3638,6 +3679,8 @@ static int octeon_usb_probe(struct platform_device *pdev) usb = (struct octeon_hcd *)hcd->hcd_priv; spin_lock_init(&usb->lock); + hrtimer_setup(&usb->sof_timer, octeon_usb_sof_timer, CLOCK_MONOTONIC, + HRTIMER_MODE_REL); usb->init_flags = initialize_flags; @@ -3688,6 +3731,7 @@ static void octeon_usb_remove(struct platform_device *pdev) unsigned long flags; usb_remove_hcd(hcd); + hrtimer_cancel(&usb->sof_timer); spin_lock_irqsave(&usb->lock, flags); status = cvmx_usb_shutdown(usb); spin_unlock_irqrestore(&usb->lock, flags); diff --git a/drivers/usb/host/ohci-da8xx.c b/drivers/usb/host/ohci-da8xx.c index 0938c0e7a8b6..a7cd5fb4f119 100644 --- a/drivers/usb/host/ohci-da8xx.c +++ b/drivers/usb/host/ohci-da8xx.c @@ -447,6 +447,7 @@ static int ohci_da8xx_probe(struct platform_device *pdev) err_remove_hcd: usb_remove_hcd(hcd); + device_wakeup_disable(hcd->self.controller); err: usb_put_hcd(hcd); return error; @@ -457,6 +458,7 @@ static void ohci_da8xx_remove(struct platform_device *pdev) struct usb_hcd *hcd = platform_get_drvdata(pdev); usb_remove_hcd(hcd); + device_wakeup_disable(hcd->self.controller); usb_put_hcd(hcd); } diff --git a/drivers/usb/host/ohci-s3c2410.c b/drivers/usb/host/ohci-s3c2410.c index e623e24d3f8e..7c0e35be58f1 100644 --- a/drivers/usb/host/ohci-s3c2410.c +++ b/drivers/usb/host/ohci-s3c2410.c @@ -335,6 +335,7 @@ ohci_hcd_s3c2410_remove(struct platform_device *dev) struct usb_hcd *hcd = platform_get_drvdata(dev); usb_remove_hcd(hcd); + device_wakeup_disable(hcd->self.controller); s3c2410_stop_hc(dev); usb_put_hcd(hcd); } diff --git a/drivers/usb/host/ohci-spear.c b/drivers/usb/host/ohci-spear.c index 6843d7cb3f9f..5aeabb051265 100644 --- a/drivers/usb/host/ohci-spear.c +++ b/drivers/usb/host/ohci-spear.c @@ -103,6 +103,7 @@ static void spear_ohci_hcd_drv_remove(struct platform_device *pdev) struct spear_ohci *sohci_p = to_spear_ohci(hcd); usb_remove_hcd(hcd); + device_wakeup_disable(hcd->self.controller); clk_disable_unprepare(sohci_p->clk); usb_put_hcd(hcd); diff --git a/drivers/usb/host/ohci-st.c b/drivers/usb/host/ohci-st.c index d1656fce5400..53010a277b67 100644 --- a/drivers/usb/host/ohci-st.c +++ b/drivers/usb/host/ohci-st.c @@ -235,6 +235,7 @@ static void st_ohci_platform_remove(struct platform_device *dev) int clk; usb_remove_hcd(hcd); + device_wakeup_disable(hcd->self.controller); if (pdata->power_off) pdata->power_off(dev); diff --git a/drivers/usb/serial/bus.c b/drivers/usb/serial/bus.c index 9e2a18c0b218..ea1fe5d1e449 100644 --- a/drivers/usb/serial/bus.c +++ b/drivers/usb/serial/bus.c @@ -165,7 +165,11 @@ int usb_serial_bus_register(struct usb_serial_driver *driver) void usb_serial_bus_deregister(struct usb_serial_driver *driver) { - free_dynids(driver); driver_unregister(&driver->driver); + free_dynids(driver); } +void usb_serial_bus_remove_new_id(struct usb_serial_driver *driver) +{ + driver_remove_file(&driver->driver, &driver_attr_new_id); +} diff --git a/drivers/usb/serial/cp210x.c b/drivers/usb/serial/cp210x.c index 333b8b463704..c3f9add98333 100644 --- a/drivers/usb/serial/cp210x.c +++ b/drivers/usb/serial/cp210x.c @@ -223,6 +223,7 @@ static const struct usb_device_id id_table[] = { { USB_DEVICE(0x19CF, 0x3000) }, /* Parrot NMEA GPS Flight Recorder */ { USB_DEVICE(0x1ADB, 0x0001) }, /* Schweitzer Engineering C662 Cable */ { USB_DEVICE(0x1B1C, 0x1C00) }, /* Corsair USB Dongle */ + { USB_DEVICE(0x1B1C, 0x1C02) }, /* Corsair AX1500i Power Supply */ { USB_DEVICE(0x1B93, 0x1013) }, /* Phoenix Contact UPS Device */ { USB_DEVICE(0x1BA4, 0x0002) }, /* Silicon Labs 358x factory default */ { USB_DEVICE(0x1BE3, 0x07A6) }, /* WAGO 750-923 USB Service Cable */ diff --git a/drivers/usb/serial/generic.c b/drivers/usb/serial/generic.c index 6eaf74930aa3..17272701fab0 100644 --- a/drivers/usb/serial/generic.c +++ b/drivers/usb/serial/generic.c @@ -266,6 +266,7 @@ EXPORT_SYMBOL_GPL(usb_serial_generic_chars_in_buffer); void usb_serial_generic_wait_until_sent(struct tty_struct *tty, long timeout) { struct usb_serial_port *port = tty->driver_data; + struct tty_port *tport = &port->port; unsigned int bps; unsigned long period; unsigned long expire; @@ -285,7 +286,14 @@ void usb_serial_generic_wait_until_sent(struct tty_struct *tty, long timeout) __func__, jiffies_to_msecs(timeout), jiffies_to_msecs(period)); expire = jiffies + timeout; - while (!port->serial->type->tx_empty(port)) { + for (;;) { + mutex_lock(&tport->mutex); + if (tty_io_error(tty) || port->serial->type->tx_empty(port)) { + mutex_unlock(&tport->mutex); + break; + } + mutex_unlock(&tport->mutex); + schedule_timeout_interruptible(period); if (signal_pending(current)) break; diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c index e4ad14375d6a..828b1be3fee2 100644 --- a/drivers/usb/serial/option.c +++ b/drivers/usb/serial/option.c @@ -260,6 +260,7 @@ static void option_instat_callback(struct urb *urb); #define QUECTEL_PRODUCT_EM060K_12a 0x012a #define QUECTEL_PRODUCT_EM060K_12b 0x012b #define QUECTEL_PRODUCT_EM060K_12c 0x012c +#define QUECTEL_PRODUCT_RG660QB 0x013d #define QUECTEL_PRODUCT_EG91 0x0191 #define QUECTEL_PRODUCT_EG95 0x0195 #define QUECTEL_PRODUCT_BG96 0x0296 @@ -280,6 +281,7 @@ static void option_instat_callback(struct urb *urb); #define QUECTEL_PRODUCT_EC200U 0x0901 #define QUECTEL_PRODUCT_EG912Y 0x6001 #define QUECTEL_PRODUCT_EC200S_CN 0x6002 +#define QUECTEL_PRODUCT_EG060W 0x6004 #define QUECTEL_PRODUCT_EC200A 0x6005 #define QUECTEL_PRODUCT_EG916Q 0x6007 #define QUECTEL_PRODUCT_EM061K_LWW 0x6008 @@ -1268,12 +1270,15 @@ static const struct usb_device_id option_ids[] = { { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EC200A, 0xff, 0, 0) }, { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EC200U, 0xff, 0, 0) }, { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EC200S_CN, 0xff, 0, 0) }, + { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EG060W, 0xff, 0, 0) }, { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EC200T, 0xff, 0, 0) }, { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EG912Y, 0xff, 0, 0) }, { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EG916Q, 0xff, 0x00, 0x00) }, { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RM500K, 0xff, 0x00, 0x00) }, { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RG650V, 0xff, 0xff, 0x30) }, { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RG650V, 0xff, 0, 0) }, + { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RG660QB, 0xff, 0xff, 0x30) }, + { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RG660QB, 0xff, 0, 0) }, { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RG255C, 0xff, 0xff, 0x30) }, { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RG255C, 0xff, 0, 0) }, { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RG255C, 0xff, 0xff, 0x40) }, @@ -2168,6 +2173,8 @@ static const struct usb_device_id option_ids[] = { { USB_DEVICE_INTERFACE_CLASS(0x1e0e, 0x9003, 0xff) }, /* Simcom SIM7500/SIM7600 MBIM mode */ { USB_DEVICE_INTERFACE_CLASS(0x1e0e, 0x9011, 0xff), /* Simcom SIM7500/SIM7600 RNDIS mode */ .driver_info = RSVD(7) }, + { USB_DEVICE_INTERFACE_CLASS(0x1e0e, 0x902b, 0xff), + .driver_info = RSVD(4) }, { USB_DEVICE(0x1e0e, 0x9071), /* Simcom SIM8230 RMNET mode */ .driver_info = RSVD(3) | RSVD(4) }, { USB_DEVICE_INTERFACE_CLASS(0x1e0e, 0x9078, 0xff), /* Simcom SIM8230 ECM mode */ @@ -2429,6 +2436,13 @@ static const struct usb_device_id option_ids[] = { .driver_info = RSVD(5) }, { USB_DEVICE_INTERFACE_CLASS(0x0489, 0xe167, 0xff), /* Foxconn T99W640 MBIM */ .driver_info = RSVD(3) }, + { USB_DEVICE_AND_INTERFACE_INFO(0x04b7, 0x4d22, 0xff, 0x10, 0x01) }, /* Compal EXC-T1 */ + { USB_DEVICE_AND_INTERFACE_INFO(0x04b7, 0x4d22, 0xff, 0x10, 0x02) }, + { USB_DEVICE_AND_INTERFACE_INFO(0x04b7, 0x4d22, 0xff, 0x10, 0x03) }, + { USB_DEVICE_AND_INTERFACE_INFO(0x04b7, 0x4d22, 0xff, 0x10, 0x04) }, + { USB_DEVICE_AND_INTERFACE_INFO(0x04b7, 0x8217, 0xff, 0xff, 0x30) }, /* Compal EXM-G1x */ + { USB_DEVICE_AND_INTERFACE_INFO(0x04b7, 0x8217, 0xff, 0xff, 0x40) }, + { USB_DEVICE_AND_INTERFACE_INFO(0x04b7, 0x8217, 0xff, 0xff, 0x60) }, { USB_DEVICE(0x1508, 0x1001), /* Fibocom NL668 (IOT version) */ .driver_info = RSVD(4) | RSVD(5) | RSVD(6) }, { USB_DEVICE(0x1782, 0x4d10) }, /* Fibocom L610 (AT mode) */ diff --git a/drivers/usb/serial/quatech2.c b/drivers/usb/serial/quatech2.c index dc2b39810a26..3b4df3ac7c73 100644 --- a/drivers/usb/serial/quatech2.c +++ b/drivers/usb/serial/quatech2.c @@ -52,6 +52,7 @@ #define QT2_XMIT_FLUSH 0x05 /* no following info */ #define QT2_CONTROL_ESCAPE 0xff /* pass through previous 2 control bytes */ +#define MIN_BAUD_RATE 50 #define MAX_BAUD_RATE 921600 #define DEFAULT_BAUD_RATE 9600 @@ -156,7 +157,7 @@ static inline int calc_baud_divisor(int baudrate) static inline int qt2_set_port_config(struct usb_device *dev, unsigned char port_number, - u16 baudrate, u16 lcr) + speed_t baudrate, u16 lcr) { int divisor = calc_baud_divisor(baudrate); u16 index = ((u16) (lcr << 8) | (u16) (port_number)); @@ -257,9 +258,9 @@ static void qt2_set_termios(struct tty_struct *tty, struct usb_device *dev = port->serial->dev; struct qt2_port_private *port_priv; struct ktermios *termios = &tty->termios; - u16 baud; unsigned int cflag = termios->c_cflag; u16 new_lcr = 0; + speed_t baud; int status; port_priv = usb_get_serial_port_data(port); @@ -277,6 +278,18 @@ static void qt2_set_termios(struct tty_struct *tty, if (!baud) baud = 9600; + if (baud < MIN_BAUD_RATE || baud > MAX_BAUD_RATE) { + if (old_termios) + baud = tty_termios_baud_rate(old_termios); + else + baud = clamp(baud, MIN_BAUD_RATE, MAX_BAUD_RATE); + + tty_encode_baud_rate(tty, baud, baud); + + if (!baud) + baud = 9600; + } + status = qt2_set_port_config(dev, port_priv->device_port, baud, new_lcr); if (status < 0) @@ -373,7 +386,7 @@ static int qt2_open(struct tty_struct *tty, struct usb_serial_port *port) port_priv->device_port = (u8) device_port; if (tty) - qt2_set_termios(tty, port, &tty->termios); + qt2_set_termios(tty, port, NULL); return 0; diff --git a/drivers/usb/serial/ssu100.c b/drivers/usb/serial/ssu100.c index b0d51558b73c..8c5b9ef708b3 100644 --- a/drivers/usb/serial/ssu100.c +++ b/drivers/usb/serial/ssu100.c @@ -32,6 +32,7 @@ #define SERIAL_EVEN_PARITY (UART_LCR_PARITY | UART_LCR_EPAR) +#define MIN_BAUD_RATE 50 #define MAX_BAUD_RATE 460800 #define ATC_DISABLED 0x00 @@ -217,9 +218,10 @@ static void ssu100_set_termios(struct tty_struct *tty, { struct usb_device *dev = port->serial->dev; struct ktermios *termios = &tty->termios; - u16 baud, divisor, remainder; + speed_t baud, remainder; unsigned int cflag = termios->c_cflag; u16 urb_value = 0; /* will hold the new flags */ + u16 divisor; int result; if (cflag & PARENB) { @@ -235,6 +237,18 @@ static void ssu100_set_termios(struct tty_struct *tty, if (!baud) baud = 9600; + if (baud < MIN_BAUD_RATE || baud > MAX_BAUD_RATE) { + if (old_termios) + baud = tty_termios_baud_rate(old_termios); + else + baud = clamp(baud, MIN_BAUD_RATE, MAX_BAUD_RATE); + + tty_encode_baud_rate(tty, baud, baud); + + if (!baud) + baud = 9600; + } + dev_dbg(&port->dev, "%s - got baud = %d\n", __func__, baud); @@ -310,7 +324,7 @@ static int ssu100_open(struct tty_struct *tty, struct usb_serial_port *port) dev_dbg(&port->dev, "%s - set uart failed\n", __func__); if (tty) - ssu100_set_termios(tty, port, &tty->termios); + ssu100_set_termios(tty, port, NULL); return usb_serial_generic_open(tty, port); } diff --git a/drivers/usb/serial/usb-serial.c b/drivers/usb/serial/usb-serial.c index 17edc057a311..4a19621f08ba 100644 --- a/drivers/usb/serial/usb-serial.c +++ b/drivers/usb/serial/usb-serial.c @@ -400,17 +400,13 @@ static unsigned int serial_chars_in_buffer(struct tty_struct *tty) static void serial_wait_until_sent(struct tty_struct *tty, int timeout) { struct usb_serial_port *port = tty->driver_data; - struct usb_serial *serial = port->serial; dev_dbg(&port->dev, "%s\n", __func__); if (!port->serial->type->wait_until_sent) return; - mutex_lock(&serial->disc_mutex); - if (!serial->disconnected) - port->serial->type->wait_until_sent(tty, timeout); - mutex_unlock(&serial->disc_mutex); + port->serial->type->wait_until_sent(tty, timeout); } static void serial_throttle(struct tty_struct *tty) @@ -438,8 +434,13 @@ static int serial_get_serial(struct tty_struct *tty, struct serial_struct *ss) struct usb_serial_port *port = tty->driver_data; struct tty_port *tport = &port->port; unsigned int close_delay, closing_wait; + int ret = 0; mutex_lock(&tport->mutex); + if (tty_io_error(tty)) { + ret = -EIO; + goto out_unlock; + } close_delay = jiffies_to_msecs(tport->close_delay) / 10; closing_wait = tport->closing_wait; @@ -452,10 +453,10 @@ static int serial_get_serial(struct tty_struct *tty, struct serial_struct *ss) if (port->serial->type->get_serial) port->serial->type->get_serial(tty, ss); - +out_unlock: mutex_unlock(&tport->mutex); - return 0; + return ret; } static int serial_set_serial(struct tty_struct *tty, struct serial_struct *ss) @@ -471,6 +472,10 @@ static int serial_set_serial(struct tty_struct *tty, struct serial_struct *ss) closing_wait = msecs_to_jiffies(closing_wait * 10); mutex_lock(&tport->mutex); + if (tty_io_error(tty)) { + ret = -EIO; + goto out_unlock; + } if (!capable(CAP_SYS_ADMIN)) { if (close_delay != tport->close_delay || @@ -498,6 +503,7 @@ static int serial_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned long arg) { struct usb_serial_port *port = tty->driver_data; + struct tty_port *tport = &port->port; int retval = -ENOIOCTLCMD; dev_dbg(&port->dev, "%s - cmd 0x%04x\n", __func__, cmd); @@ -508,8 +514,21 @@ static int serial_ioctl(struct tty_struct *tty, retval = port->serial->type->tiocmiwait(tty, arg); break; default: - if (port->serial->type->ioctl) + if (!port->serial->type->ioctl) + break; + + if (cmd == TIOCSRS485) + down_write(&tty->termios_rwsem); + + mutex_lock(&tport->mutex); + if (tty_io_error(tty)) + retval = -EIO; + else retval = port->serial->type->ioctl(tty, cmd, arg); + mutex_unlock(&tport->mutex); + + if (cmd == TIOCSRS485) + up_write(&tty->termios_rwsem); } return retval; @@ -519,25 +538,40 @@ static void serial_set_termios(struct tty_struct *tty, const struct ktermios *old) { struct usb_serial_port *port = tty->driver_data; + struct tty_port *tport = &port->port; dev_dbg(&port->dev, "%s\n", __func__); - if (port->serial->type->set_termios) - port->serial->type->set_termios(tty, port, old); - else + if (!port->serial->type->set_termios) { tty_termios_copy_hw(&tty->termios, old); + return; + } + + mutex_lock(&tport->mutex); + if (!tty_io_error(tty)) + port->serial->type->set_termios(tty, port, old); + mutex_unlock(&tport->mutex); } static int serial_break(struct tty_struct *tty, int break_state) { struct usb_serial_port *port = tty->driver_data; + struct tty_port *tport = &port->port; + int ret; dev_dbg(&port->dev, "%s\n", __func__); - if (port->serial->type->break_ctl) - return port->serial->type->break_ctl(tty, break_state); + if (!port->serial->type->break_ctl) + return -ENOTTY; - return -ENOTTY; + mutex_lock(&tport->mutex); + if (tty_io_error(tty)) + ret = -EIO; + else + ret = port->serial->type->break_ctl(tty, break_state); + mutex_unlock(&tport->mutex); + + return ret; } static int serial_proc_show(struct seq_file *m, void *v) @@ -578,24 +612,44 @@ static int serial_proc_show(struct seq_file *m, void *v) static int serial_tiocmget(struct tty_struct *tty) { struct usb_serial_port *port = tty->driver_data; + struct tty_port *tport = &port->port; + int ret; dev_dbg(&port->dev, "%s\n", __func__); - if (port->serial->type->tiocmget) - return port->serial->type->tiocmget(tty); - return -ENOTTY; + if (!port->serial->type->tiocmget) + return -ENOTTY; + + mutex_lock(&tport->mutex); + if (tty_io_error(tty)) + ret = -EIO; + else + ret = port->serial->type->tiocmget(tty); + mutex_unlock(&tport->mutex); + + return ret; } static int serial_tiocmset(struct tty_struct *tty, unsigned int set, unsigned int clear) { struct usb_serial_port *port = tty->driver_data; + struct tty_port *tport = &port->port; + int ret; dev_dbg(&port->dev, "%s\n", __func__); - if (port->serial->type->tiocmset) - return port->serial->type->tiocmset(tty, set, clear); - return -ENOTTY; + if (!port->serial->type->tiocmset) + return -ENOTTY; + + mutex_lock(&tport->mutex); + if (tty_io_error(tty)) + ret = -EIO; + else + ret = port->serial->type->tiocmset(tty, set, clear); + mutex_unlock(&tport->mutex); + + return ret; } static int serial_get_icount(struct tty_struct *tty, @@ -1191,11 +1245,16 @@ static void usb_serial_disconnect(struct usb_interface *interface) usb_serial_port_poison_urbs(port); wake_up_interruptible(&port->port.delta_msr_wait); cancel_work_sync(&port->work); + } + + if (serial->type->disconnect) + serial->type->disconnect(serial); + + for (i = 0; i < serial->num_ports; ++i) { + port = serial->port[i]; if (device_is_registered(&port->dev)) device_del(&port->dev); } - if (serial->type->disconnect) - serial->type->disconnect(serial); release_sibling(serial, interface); @@ -1313,7 +1372,8 @@ static int __init usb_serial_init(void) int result; usb_serial_tty_driver = tty_alloc_driver(USB_SERIAL_TTY_MINORS, - TTY_DRIVER_REAL_RAW | TTY_DRIVER_DYNAMIC_DEV); + TTY_DRIVER_REAL_RAW | TTY_DRIVER_DYNAMIC_DEV | + TTY_DRIVER_RESET_SAVED_TERMIOS); if (IS_ERR(usb_serial_tty_driver)) return PTR_ERR(usb_serial_tty_driver); @@ -1464,7 +1524,7 @@ int __usb_serial_register_drivers(struct usb_serial_driver *const serial_drivers { int rc; struct usb_driver *udriver; - struct usb_serial_driver * const *sd; + struct usb_serial_driver * const *sd, * const *s; /* * udriver must be registered before any of the serial drivers, @@ -1517,9 +1577,11 @@ int __usb_serial_register_drivers(struct usb_serial_driver *const serial_drivers return 0; err_deregister_drivers: + for (s = serial_drivers; s < sd; ++s) + usb_serial_bus_remove_new_id(*s); + usb_deregister(udriver); while (sd-- > serial_drivers) usb_serial_deregister(*sd); - usb_deregister(udriver); err_free_driver: kfree(udriver); return rc; @@ -1537,10 +1599,23 @@ EXPORT_SYMBOL_GPL(__usb_serial_register_drivers); void usb_serial_deregister_drivers(struct usb_serial_driver *const serial_drivers[]) { struct usb_driver *udriver = (*serial_drivers)->usb_driver; + struct usb_serial_driver * const *sd; + + /* + * udriver must be deregistered before the serial drivers so that + * I/O is stopped before unbinding the ports. + * + * Remove the new_id attributes to prevent ids from being added and + * triggering a probe of udriver after it has been deregistered. + */ + for (sd = serial_drivers; *sd; ++sd) + usb_serial_bus_remove_new_id(*sd); - for (; *serial_drivers; ++serial_drivers) - usb_serial_deregister(*serial_drivers); usb_deregister(udriver); + + for (sd = serial_drivers; *sd; ++sd) + usb_serial_deregister(*sd); + kfree(udriver); } EXPORT_SYMBOL_GPL(usb_serial_deregister_drivers); diff --git a/drivers/usb/serial/xr_serial.c b/drivers/usb/serial/xr_serial.c index 352c765d8803..c08f4aa14a3d 100644 --- a/drivers/usb/serial/xr_serial.c +++ b/drivers/usb/serial/xr_serial.c @@ -850,12 +850,9 @@ static int xr_get_rs485_config(struct tty_struct *tty, struct usb_serial_port *port = tty->driver_data; struct xr_data *data = usb_get_serial_port_data(port); - down_read(&tty->termios_rwsem); - if (copy_to_user(argp, &data->rs485, sizeof(data->rs485))) { - up_read(&tty->termios_rwsem); + /* core holds port mutex */ + if (copy_to_user(argp, &data->rs485, sizeof(data->rs485))) return -EFAULT; - } - up_read(&tty->termios_rwsem); return 0; } @@ -871,10 +868,9 @@ static int xr_set_rs485_config(struct tty_struct *tty, return -EFAULT; xr_sanitize_serial_rs485(&rs485); - down_write(&tty->termios_rwsem); + /* core holds termios rwsem and port mutex */ data->rs485 = rs485; xr_set_flow_mode(tty, port, NULL); - up_write(&tty->termios_rwsem); if (copy_to_user(argp, &rs485, sizeof(rs485))) return -EFAULT; diff --git a/drivers/usb/storage/sierra_ms.c b/drivers/usb/storage/sierra_ms.c index c5e3eeeb2144..992685ca6f1a 100644 --- a/drivers/usb/storage/sierra_ms.c +++ b/drivers/usb/storage/sierra_ms.c @@ -77,6 +77,13 @@ static int sierra_get_swoc_info(struct usb_device *udev, sizeof(struct swoc_info), /* __u16 size */ USB_CTRL_SET_TIMEOUT); /* int timeout */ + /* + * A short IN transfer leaves the tail of swocInfo uninitialized; + * only a full transfer is valid. + */ + if (result != sizeof(struct swoc_info)) + return -EIO; + swocInfo->LinuxSKU = le16_to_cpu(swocInfo->LinuxSKU); swocInfo->LinuxVer = le16_to_cpu(swocInfo->LinuxVer); return result; diff --git a/drivers/usb/typec/anx7411.c b/drivers/usb/typec/anx7411.c index 41df115912b9..c3d36da692e4 100644 --- a/drivers/usb/typec/anx7411.c +++ b/drivers/usb/typec/anx7411.c @@ -1566,8 +1566,11 @@ static void anx7411_i2c_remove(struct i2c_client *client) anx7411_partner_unregister_altmode(plat); anx7411_unregister_partner(plat); - if (plat->workqueue) + if (plat->workqueue) { + disable_irq(plat->intp_irq); + cancel_work_sync(&plat->work); destroy_workqueue(plat->workqueue); + } i2c_unregister_device(plat->spi_client); diff --git a/drivers/usb/typec/port-mapper.c b/drivers/usb/typec/port-mapper.c index cdbb7c11d714..3a97b084c3bd 100644 --- a/drivers/usb/typec/port-mapper.c +++ b/drivers/usb/typec/port-mapper.c @@ -42,6 +42,23 @@ static int usb4_port_compare(struct device *dev, void *fwnode) return usb4_usb3_port_match(dev, fwnode); } +static bool typec_has_usb4_host_interface(const struct fwnode_handle *fwnode) +{ + if (!IS_REACHABLE(CONFIG_USB4)) + return false; + + struct fwnode_handle *nhi_fwnode __free(fwnode_handle) = + fwnode_find_reference(fwnode, "usb4-host-interface", 0); + + /* + * The USB4 port can only appear if the host interface is enabled in + * the firmware and has been enumerated as a device. The latter is + * the same check as in usb_acpi_add_usb4_devlink(). + */ + return !IS_ERR(nhi_fwnode) && fwnode_device_is_available(nhi_fwnode) && + nhi_fwnode->dev; +} + static int typec_port_compare(struct device *dev, void *fwnode) { return device_match_fwnode(dev, fwnode); @@ -64,11 +81,11 @@ static int typec_port_match(struct device *dev, void *data) adev_fwnode); /* - * If dev is USB 3.x port, it may have reference to the + * If dev is USB 3.x port, it may have reference to an available * USB4 host interface in which case we can also link the * Type-C port with the USB4 port. */ - if (fwnode_property_present(adev_fwnode, "usb4-host-interface")) + if (typec_has_usb4_host_interface(adev_fwnode)) component_match_add(&arg->port->dev, &arg->match, usb4_port_compare, adev_fwnode); } diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index 2d6b14aa2085..06692feb6d46 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -1756,7 +1756,8 @@ static void mod_enable_frs_delayed_work(struct tcpm_port *port, unsigned int del static void mod_vdm_discovery_cancel_delayed_work(struct tcpm_port *port) { hrtimer_cancel(&port->vdm_discovery_timer); - kthread_cancel_work_sync(&port->vdm_discovery_work); + if (port->wq) + kthread_cancel_work_sync(&port->vdm_discovery_work); } static void mod_vdm_discovery_delayed_work(struct tcpm_port *port, unsigned int delay_ms) @@ -7113,7 +7114,7 @@ static void tcpm_pd_event_handler(struct kthread_work *work) port->upcoming_state = FR_SWAP_SEND; ret = tcpm_ams_start(port, FAST_ROLE_SWAP); if (ret == -EAGAIN) - port->upcoming_state = INVALID_STATE; + tcpm_set_state(port, ERROR_RECOVERY, 0); } else { tcpm_log(port, "Discarding FRS_SIGNAL! Not in sink ready"); } @@ -8977,6 +8978,7 @@ void tcpm_unregister_port(struct tcpm_port *port) port->registered = false; kthread_destroy_worker(port->wq); + port->wq = NULL; hrtimer_cancel(&port->vdm_discovery_timer); hrtimer_cancel(&port->enable_frs_timer); diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index f76f563dc42b..483539185177 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -343,7 +343,7 @@ static void tps6598x_set_data_role(struct tps6598x *tps, static int tps6598x_connect(struct tps6598x *tps, u32 status) { - struct typec_partner_desc desc; + struct typec_partner_desc desc = { }; enum typec_pwr_opmode mode; int ret; @@ -354,7 +354,6 @@ static int tps6598x_connect(struct tps6598x *tps, u32 status) desc.usb_pd = mode == TYPEC_PWR_MODE_PD; desc.accessory = TYPEC_ACCESSORY_NONE; /* XXX: handle accessories */ - desc.identity = NULL; if (desc.usb_pd) { ret = tps6598x_read_partner_identity(tps); @@ -850,11 +849,10 @@ static void cd321x_update_work(struct work_struct *work) /* Set up partner if we were previously disconnected (or changed). */ if (!tps->partner) { - struct typec_partner_desc desc; + struct typec_partner_desc desc = { }; desc.usb_pd = is_pd; desc.accessory = TYPEC_ACCESSORY_NONE; /* XXX: handle accessories */ - desc.identity = NULL; if (desc.usb_pd) desc.identity = &st.partner_identity; @@ -1792,6 +1790,7 @@ static int tps6598x_probe(struct i2c_client *client) const struct tipd_data *data; struct tps6598x *tps; struct fwnode_handle *fwnode; + bool patch_loaded = false; u32 status; u32 vid = 0; int ret; @@ -1847,6 +1846,7 @@ static int tps6598x_probe(struct i2c_client *client) return ret; if (ret == TPS_MODE_PTCH) { + patch_loaded = true; ret = tps->data->init(tps); if (ret) return ret; @@ -1937,7 +1937,8 @@ static int tps6598x_probe(struct i2c_client *client) tps6598x_write64(tps, TPS_REG_INT_MASK1, 0); err_reset_controller: /* Reset PD controller to remove any applied patch */ - tps->data->reset(tps); + if (patch_loaded) + tps->data->reset(tps); return ret; } diff --git a/drivers/usb/typec/ucsi/displayport.c b/drivers/usb/typec/ucsi/displayport.c index 8d2032d0762c..572da7bbd29c 100644 --- a/drivers/usb/typec/ucsi/displayport.c +++ b/drivers/usb/typec/ucsi/displayport.c @@ -71,11 +71,14 @@ static int ucsi_displayport_enter(struct typec_altmode *alt, u32 *vdo) if (ret < 0) { if (ucsi->version > 0x0100) goto err_unlock; - cur = 0xff; } - if (cur < UCSI_MAX_ALTMODES) { - ret = dp->con->port_altmode[cur] == alt ? 0 : -EBUSY; + if (cur != 0xff) { + if (cur < UCSI_MAX_ALTMODES) + ret = dp->con->port_altmode[cur] == alt ? 0 : -EBUSY; + else + ret = -EINVAL; + goto err_unlock; } diff --git a/drivers/usb/typec/ucsi/ucsi.c b/drivers/usb/typec/ucsi/ucsi.c index bef3f9b71d71..c1450639c795 100644 --- a/drivers/usb/typec/ucsi/ucsi.c +++ b/drivers/usb/typec/ucsi/ucsi.c @@ -1801,10 +1801,21 @@ static struct fwnode_handle *ucsi_find_fwnode(struct ucsi_connector *con) { struct fwnode_handle *fwnode; int i = 1; + int ret; + u32 port; - device_for_each_child_node(con->ucsi->dev, fwnode) - if (i++ == con->num) - return fwnode; + device_for_each_child_node(con->ucsi->dev, fwnode) { + ret = fwnode_property_read_u32(fwnode, "reg", &port); + if (ret < 0) { + if (i == con->num) + return fwnode; + } else { + if (port == con->num - 1) + return fwnode; + } + + i++; + } return NULL; } diff --git a/drivers/usb/typec/ucsi/ucsi_acpi.c b/drivers/usb/typec/ucsi/ucsi_acpi.c index 18286d3e9cc5..5697ccc24ec3 100644 --- a/drivers/usb/typec/ucsi/ucsi_acpi.c +++ b/drivers/usb/typec/ucsi/ucsi_acpi.c @@ -121,6 +121,33 @@ static const struct ucsi_operations ucsi_acpi_ops = { .async_control = ucsi_acpi_async_control }; +static int ucsi_acer_read_version(struct ucsi *ucsi, u16 *version) +{ + struct ucsi_acpi *ua = ucsi_get_drvdata(ucsi); + int ret; + + ret = ucsi_acpi_read_version(ucsi, version); + if (ret) + return ret; + + if (!*version) { + dev_warn(ua->dev, "UCSI version is zero, assuming 1.2\n"); + *version = UCSI_VERSION_1_2; + } + + return 0; +} + +static const struct ucsi_operations ucsi_acer_ops = { + .read_version = ucsi_acer_read_version, + .read_cci = ucsi_acpi_read_cci, + .poll_cci = ucsi_acpi_poll_cci, + .read_message_in = ucsi_acpi_read_message_in, + .write_message_out = ucsi_acpi_write_message_out, + .sync_control = ucsi_sync_control_common, + .async_control = ucsi_acpi_async_control +}; + static int ucsi_gram_sync_control(struct ucsi *ucsi, u64 command, u32 *cci, void *val, size_t len, void *msg_out, size_t msg_out_size) @@ -162,6 +189,13 @@ static const struct ucsi_operations ucsi_gram_ops = { }; static const struct dmi_system_id ucsi_acpi_quirks[] = { + { + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Acer"), + DMI_MATCH(DMI_PRODUCT_NAME, "Nitro ANV15-41"), + }, + .driver_data = (void *)&ucsi_acer_ops, + }, { .matches = { DMI_MATCH(DMI_SYS_VENDOR, "LG Electronics"), diff --git a/drivers/usb/typec/ucsi/ucsi_glink.c b/drivers/usb/typec/ucsi/ucsi_glink.c index 12e07b9fe622..1db0c88d817e 100644 --- a/drivers/usb/typec/ucsi/ucsi_glink.c +++ b/drivers/usb/typec/ucsi/ucsi_glink.c @@ -362,6 +362,10 @@ static void pmic_glink_ucsi_destroy(void *data) { struct pmic_glink_ucsi *ucsi = data; + /* Drain the work items before ucsi_destroy() frees the ucsi instance */ + cancel_work_sync(&ucsi->notify_work); + cancel_work_sync(&ucsi->register_work); + /* Protect to make sure we're not in a middle of a transaction from a glink callback */ mutex_lock(&ucsi->lock); ucsi_destroy(ucsi->ucsi); @@ -467,8 +471,15 @@ static void pmic_glink_ucsi_remove(struct auxiliary_device *adev) { struct pmic_glink_ucsi *ucsi = dev_get_drvdata(&adev->dev); - /* Unregister first to stop having read & writes */ - ucsi_unregister(ucsi->ucsi); + /* Callbacks can queue work until devres releases the client */ + disable_work_sync(&ucsi->notify_work); + disable_work_sync(&ucsi->register_work); + + /* register_work may have unregistered the instance already */ + if (ucsi->ucsi_registered) { + ucsi->ucsi_registered = false; + ucsi_unregister(ucsi->ucsi); + } } static const struct auxiliary_device_id pmic_glink_ucsi_id_table[] = { diff --git a/drivers/virt/nitro_enclaves/ne_misc_dev.c b/drivers/virt/nitro_enclaves/ne_misc_dev.c index c91300a73f50..0d7bdad924b7 100644 --- a/drivers/virt/nitro_enclaves/ne_misc_dev.c +++ b/drivers/virt/nitro_enclaves/ne_misc_dev.c @@ -1706,6 +1706,7 @@ static int ne_create_vm_ioctl(struct ne_pci_dev *ne_pci_dev, u64 __user *slot_ui return enclave_fd; put_file: + enclave_file->private_data = NULL; fput(enclave_file); put_fd: put_unused_fd(enclave_fd); diff --git a/drivers/virt/vmgenid.c b/drivers/virt/vmgenid.c index 66135eac3abf..ce957f5b1c31 100644 --- a/drivers/virt/vmgenid.c +++ b/drivers/virt/vmgenid.c @@ -25,7 +25,7 @@ struct vmgenid_state { static void vmgenid_notify(struct device *device) { - struct vmgenid_state *state = device->driver_data; + struct vmgenid_state *state = dev_get_drvdata(device); u8 old_id[VMGENID_SIZE]; memcpy(old_id, state->this_id, sizeof(old_id)); @@ -75,13 +75,16 @@ static int vmgenid_add_acpi(struct device *dev, struct vmgenid_state *state) phys_addr = (obj->package.elements[0].integer.value << 0) | (obj->package.elements[1].integer.value << 32); - virt_addr = devm_memremap(&device->dev, phys_addr, VMGENID_SIZE, MEMREMAP_WB); + virt_addr = devm_memremap(&device->dev, phys_addr, VMGENID_SIZE, + MEMREMAP_WB | MEMREMAP_DEC); if (IS_ERR(virt_addr)) { ret = PTR_ERR(virt_addr); goto out; } setup_vmgenid_state(state, virt_addr); + dev_set_drvdata(dev, state); + status = acpi_install_notify_handler(device->handle, ACPI_DEVICE_NOTIFY, vmgenid_acpi_handler, dev); if (ACPI_FAILURE(status)) { @@ -89,7 +92,6 @@ static int vmgenid_add_acpi(struct device *dev, struct vmgenid_state *state) goto out; } - dev->driver_data = state; out: ACPI_FREE(parsed.pointer); return ret; @@ -123,12 +125,13 @@ static int vmgenid_add_of(struct platform_device *pdev, if (ret < 0) return ret; + dev_set_drvdata(&pdev->dev, state); + ret = devm_request_irq(&pdev->dev, ret, vmgenid_of_irq_handler, IRQF_SHARED, "vmgenid", &pdev->dev); if (ret < 0) return ret; - pdev->dev.driver_data = state; return 0; } diff --git a/fs/autofs/inode.c b/fs/autofs/inode.c index 6b15a3717ba7..066c16f2ea56 100644 --- a/fs/autofs/inode.c +++ b/fs/autofs/inode.c @@ -51,6 +51,10 @@ void autofs_kill_sb(struct super_block *sb) if (sbi) { /* Free wait queues, close pipe */ autofs_catatonic_mode(sbi); + if (sbi->pipe) { + fput(sbi->pipe); + sbi->pipe = NULL; + } put_pid(sbi->oz_pgrp); } diff --git a/fs/binfmt_misc_bpf.c b/fs/binfmt_misc_bpf.c index 91576ff05911..a3e26e8a4027 100644 --- a/fs/binfmt_misc_bpf.c +++ b/fs/binfmt_misc_bpf.c @@ -141,8 +141,6 @@ __bpf_kfunc int bpf_binprm_set_interp(struct linux_binprm *bprm, len = strnlen(path, path__sz); if (len == path__sz) return -EINVAL; - if (path[0] != '/') - return -EINVAL; if (len >= PATH_MAX) return -ENAMETOOLONG; @@ -150,6 +148,15 @@ __bpf_kfunc int bpf_binprm_set_interp(struct linux_binprm *bprm, if (!interp) return -ENOMEM; + /* + * The program may pass memory that is written to while this runs, + * so check the private copy and not the buffer it was made from. + */ + if (interp[0] != '/') { + kfree(interp); + return -EINVAL; + } + bm_bpf_stage_selection(bprm, interp, NULL); return 0; } @@ -176,6 +183,7 @@ __bpf_kfunc int bpf_binprm_select_interp(struct linux_binprm *bprm, const char *name, size_t name__sz) { const struct binfmt_misc_interp *interp; + char buf[BINFMT_MISC_INTERP_NAME_MAX + 1]; size_t len; char *path; @@ -184,8 +192,20 @@ __bpf_kfunc int bpf_binprm_select_interp(struct linux_binprm *bprm, len = strnlen(name, name__sz); if (len == name__sz || !len) return -EINVAL; + /* No entry binds a longer name, so it cannot be found. */ + if (len > BINFMT_MISC_INTERP_NAME_MAX) + return -ENOENT; - interp = binfmt_misc_find_interp(bprm->bpf_interps, name); + /* + * The program may pass memory that is written to while this runs, + * so look the name up in a private copy and check that instead. + */ + memcpy(buf, name, len); + buf[len] = '\0'; + if (!buf[0]) + return -EINVAL; + + interp = binfmt_misc_find_interp(bprm->bpf_interps, buf); if (!interp) return -ENOENT; @@ -228,6 +248,15 @@ __bpf_kfunc int bpf_binprm_set_interp_arg(struct linux_binprm *bprm, if (!val) return -ENOMEM; + /* + * The program may pass memory that is written to while this runs, + * so check the private copy and not the buffer it was made from. + */ + if (!val[0]) { + kfree(val); + return -EINVAL; + } + kfree(bprm->bpf_interp_arg); bprm->bpf_interp_arg = val; return 0; diff --git a/fs/bpf_fs_kfuncs.c b/fs/bpf_fs_kfuncs.c index 6cb877267978..357a379ef92a 100644 --- a/fs/bpf_fs_kfuncs.c +++ b/fs/bpf_fs_kfuncs.c @@ -472,10 +472,6 @@ BTF_ID(func, bpf_lsm_inode_rmdir) BTF_ID(func, bpf_lsm_inode_setattr) BTF_ID(func, bpf_lsm_inode_setxattr) BTF_ID(func, bpf_lsm_inode_unlink) -#ifdef CONFIG_SECURITY_PATH -BTF_ID(func, bpf_lsm_path_unlink) -BTF_ID(func, bpf_lsm_path_rmdir) -#endif /* CONFIG_SECURITY_PATH */ BTF_SET_END(d_inode_locked_hooks) bool bpf_lsm_has_d_inode_locked(const struct bpf_prog *prog) diff --git a/fs/dcache.c b/fs/dcache.c index 1b1a81f10da6..a66be85f9d01 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -1916,6 +1916,10 @@ static struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name) * be overwriting an internal NUL character */ dentry->d_shortname.string[DNAME_INLINE_LEN-1] = 0; + + /* Racy __d_lookup_rcu() walk may read past the NUL; harmless */ + kmsan_unpoison_memory(dentry->d_shortname.string, DNAME_INLINE_LEN); + if (unlikely(!name)) { name = &slash_name; dname = dentry->d_shortname.string; diff --git a/fs/debugfs/inode.c b/fs/debugfs/inode.c index e054e62919ec..a4d08bd3743b 100644 --- a/fs/debugfs/inode.c +++ b/fs/debugfs/inode.c @@ -368,6 +368,9 @@ static struct dentry *debugfs_start_creating(const char *name, if (!debugfs_enabled) return ERR_PTR(-EPERM); + if (IS_ERR(parent)) + return parent; + if (!debugfs_initialized()) { pr_err("Unable to create file '%s', debugfs is not initialized yet\n", name); @@ -376,9 +379,6 @@ static struct dentry *debugfs_start_creating(const char *name, pr_debug("creating file '%s'\n", name); - if (IS_ERR(parent)) - return parent; - error = simple_pin_fs(&debug_fs_type, &debugfs_mount, &debugfs_mount_count); if (error) { diff --git a/fs/exec.c b/fs/exec.c index 819643408e6d..a5269b5e00df 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -882,6 +882,7 @@ static int exec_mmap(struct linux_binprm *bprm) active_mm = tsk->active_mm; tsk->active_mm = mm; tsk->mm = mm; + sched_cache_exec_mmap(tsk, mm); mm_init_cid(mm, tsk); exec_state = task_exec_state_replace(tsk, exec_state); /* diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index e744f9f9d43f..ea3eb40bf828 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c @@ -727,19 +727,34 @@ static bool isw_prepare_wbs_switch(struct bdi_writeback *new_wb, struct inode_switch_wbs_context *isw, struct list_head *list, int *nr) { - struct inode *inode; + struct inode *inode, *tmp; + LIST_HEAD(scanned); + bool full = false; + + /* + * Walk from the oldest end and move scanned inodes to the newest + * end, so the next scan resumes at unscanned inodes instead of + * re-walking an ever-growing run of prepared and skipped ones. + * For b_dirty_time this keeps the oldest unscanned inode at the + * end move_expired_inodes() picks from; b_attached is unordered. + */ + list_for_each_entry_safe_reverse(inode, tmp, list, i_io_list) { + list_move(&inode->i_io_list, &scanned); - list_for_each_entry(inode, list, i_io_list) { if (!inode_prepare_wbs_switch(inode, new_wb)) continue; isw->inodes[*nr] = inode; (*nr)++; - if (*nr >= WB_MAX_INODES_PER_ISW - 1) - return true; + if (*nr >= WB_MAX_INODES_PER_ISW - 1) { + full = true; + break; + } } - return false; + list_splice(&scanned, list); + + return full; } /** diff --git a/fs/inode.c b/fs/inode.c index ba7da39be4a3..a9d37be390a1 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -880,7 +880,6 @@ void evict_inodes(struct super_block *sb) struct inode *inode; LIST_HEAD(dispose); -again: spin_lock(&sb->s_inode_list_lock); list_for_each_entry(inode, &sb->s_inodes, i_sb_list) { if (icount_read_once(inode)) @@ -899,19 +898,19 @@ void evict_inodes(struct super_block *sb) inode_state_set(inode, I_FREEING); inode_lru_list_del(inode); spin_unlock(&inode->i_lock); - list_add(&inode->i_lru, &dispose); /* - * We can have a ton of inodes to evict at unmount time given - * enough memory, check to see if we need to go to sleep for a - * bit so we don't livelock. + * Keep this inode out of dispose so it stays on s_inodes while + * the list lock is dropped. I_FREEING prevents new references + * and leaves eviction to us, so we can resume the walk from it. */ if (need_resched()) { spin_unlock(&sb->s_inode_list_lock); cond_resched(); dispose_list(&dispose); - goto again; + spin_lock(&sb->s_inode_list_lock); } + list_add(&inode->i_lru, &dispose); } spin_unlock(&sb->s_inode_list_lock); diff --git a/fs/isofs/dir.c b/fs/isofs/dir.c index c7ca7603e97a..28741251d56e 100644 --- a/fs/isofs/dir.c +++ b/fs/isofs/dir.c @@ -110,25 +110,21 @@ static int do_isofs_readdir(struct inode *inode, struct file *file, return 0; } - de = (struct iso_directory_record *) (bh->b_data + offset); - - de_len = *(unsigned char *)de; - + de = (struct iso_directory_record *)(bh->b_data + offset); /* - * If the length byte is zero, we should move on to the next - * CDROM sector. If we are at the end of the directory, we - * kick out of the while loop. + * If we are at the end of a block (or at its zero-padded + * tail), move on to the next CDROM sector. If we are at the + * end of the directory, we'll abort the while loop. */ - - if (de_len == 0) { + if (offset >= bufsize || de->length[0] == 0) { brelse(bh); bh = NULL; - ctx->pos = (ctx->pos + ISOFS_BLOCK_SIZE) & ~(ISOFS_BLOCK_SIZE - 1); - block = ctx->pos >> bufbits; + block++; + ctx->pos = (loff_t)block << bufbits; offset = 0; continue; } - + de_len = de->length[0]; block_saved = block; offset_saved = offset; offset += de_len; diff --git a/fs/isofs/namei.c b/fs/isofs/namei.c index 010682f5901a..4fba1bf7f016 100644 --- a/fs/isofs/namei.c +++ b/fs/isofs/namei.c @@ -74,18 +74,20 @@ isofs_find_entry(struct inode *dir, struct dentry *dentry, return 0; } - de = (struct iso_directory_record *) (bh->b_data + offset); - - de_len = *(unsigned char *) de; - if (!de_len) { + de = (struct iso_directory_record *)(bh->b_data + offset); + /* + * If we are at the end of the block or at its zero-padded + * tail, move to the next block. + */ + if (offset >= bufsize || de->length[0] == 0) { brelse(bh); bh = NULL; - f_pos = (f_pos + ISOFS_BLOCK_SIZE) & ~(ISOFS_BLOCK_SIZE - 1); - block = f_pos >> bufbits; + block++; + f_pos = block << bufbits; offset = 0; continue; } - + de_len = de->length[0]; block_saved = bh->b_blocknr; offset_saved = offset; offset += de_len; diff --git a/fs/kernfs/mount.c b/fs/kernfs/mount.c index f183a96778b9..a57399021c8b 100644 --- a/fs/kernfs/mount.c +++ b/fs/kernfs/mount.c @@ -434,8 +434,8 @@ void kernfs_kill_sb(struct super_block *sb) up_write(&root->kernfs_supers_rwsem); /* - * Remove the superblock from fs_supers/s_instances - * so we can't find it, before freeing kernfs_super_info. + * Mark the superblock dead so sget_fc() can't find it, + * before freeing kernfs_super_info. */ kill_anon_super(sb); kfree(info); diff --git a/fs/namespace.c b/fs/namespace.c index ae5dc64f8b45..580877e46b1a 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -6184,6 +6184,21 @@ struct mnt_namespace init_mnt_ns = { .poll = __WAIT_QUEUE_HEAD_INITIALIZER(init_mnt_ns.poll), }; +static void __init mount_rootfs_on_nullfs(struct vfsmount *mnt, + struct vfsmount *nullfs_mnt) +{ + struct path root = { + .mnt = nullfs_mnt, + .dentry = nullfs_mnt->mnt_root, + }; + + LOCK_MOUNT_EXACT(mp, &root); + if (unlikely(IS_ERR(mp.parent))) + panic("VFS: Failed to mount rootfs on nullfs"); + scoped_guard(mount_writer) + attach_mnt(real_mount(mnt), mp.parent, mp.mp); +} + static void __init init_mount_tree(void) { struct vfsmount *mnt, *nullfs_mnt; @@ -6215,15 +6230,7 @@ static void __init init_mount_tree(void) mnt_root = real_mount(nullfs_mnt); init_mnt_ns.root = mnt_root; - /* Mount mutable rootfs on top of nullfs. */ - root.mnt = nullfs_mnt; - root.dentry = nullfs_mnt->mnt_root; - - LOCK_MOUNT_EXACT(mp, &root); - if (unlikely(IS_ERR(mp.parent))) - panic("VFS: Failed to mount rootfs on nullfs"); - scoped_guard(mount_writer) - attach_mnt(real_mount(mnt), mp.parent, mp.mp); + mount_rootfs_on_nullfs(mnt, nullfs_mnt); pr_info("VFS: Finished mounting rootfs on nullfs\n"); @@ -6294,7 +6301,7 @@ void put_mnt_ns(struct mnt_namespace *ns) guard(namespace_excl)(); emptied_ns = ns; guard(mount_writer)(); - umount_tree(ns->root, UMOUNT_CONNECTED); + umount_tree(ns->root, 0); } struct vfsmount *kern_mount(struct file_system_type *type) diff --git a/fs/netfs/buffered_read.c b/fs/netfs/buffered_read.c index 424df70a5c30..e6506942aeda 100644 --- a/fs/netfs/buffered_read.c +++ b/fs/netfs/buffered_read.c @@ -482,15 +482,14 @@ static int netfs_read_gaps(struct file *file, struct folio *folio) struct netfs_group *group = netfs_folio_group(folio); struct netfs_folio *finfo = netfs_folio_info(folio); struct netfs_inode *ctx = netfs_inode(mapping->host); - struct folio *sink = NULL; - struct bio_vec *bvec; + struct bio_vec *bvec = NULL; unsigned int from = finfo->dirty_offset; unsigned int to = from + finfo->dirty_len; - unsigned int off = 0, i = 0; + unsigned int off = 0; size_t flen = folio_size(folio); size_t nr_bvec = flen / PAGE_SIZE + 2; size_t part; - int ret; + int ret, i = 0, sink_from = -1, sink_to = -1; _enter("%lx", folio->index); @@ -515,24 +514,23 @@ static int netfs_read_gaps(struct file *file, struct folio *folio) if (!bvec) goto discard; - sink = folio_alloc(GFP_KERNEL, 0); - if (!sink) { - kfree(bvec); - goto discard; - } - trace_netfs_folio(folio, netfs_folio_trace_read_gaps); - rreq->direct_bv = bvec; - rreq->direct_bv_count = nr_bvec; if (from > 0) { bvec_set_folio(&bvec[i++], folio, from, 0); off = from; } + sink_from = i; while (off < to) { + struct folio *sink = folio_alloc(GFP_KERNEL, 0); + + if (!sink) + goto discard; part = min_t(size_t, to - off, PAGE_SIZE); - bvec_set_folio(&bvec[i++], sink, part, 0); + bvec_set_folio(&bvec[i], sink, part, 0); off += part; + sink_to = i; + i++; } if (to < flen) bvec_set_folio(&bvec[i++], folio, flen - to, to); @@ -543,6 +541,13 @@ static int netfs_read_gaps(struct file *file, struct folio *folio) ret = netfs_wait_for_read(rreq); if (ret >= 0) { + if (ret < flen) { + struct iov_iter iter; + + iov_iter_bvec(&iter, ITER_DEST, bvec, i, flen); + iov_iter_advance(&iter, ret); + iov_iter_zero(flen - ret, &iter); + } if (group) folio_change_private(folio, group); else @@ -553,8 +558,10 @@ static int netfs_read_gaps(struct file *file, struct folio *folio) folio_mark_uptodate(folio); } - if (sink) - folio_put(sink); + if (sink_to >= 0) + for (; sink_from <= sink_to; sink_from++) + folio_put(bvec_folio(&bvec[sink_from])); + kfree(bvec); folio_unlock(folio); netfs_put_request(rreq, netfs_rreq_trace_put_return); return ret < 0 ? ret : 0; @@ -563,6 +570,10 @@ static int netfs_read_gaps(struct file *file, struct folio *folio) netfs_put_failed_request(rreq); alloc_error: folio_unlock(folio); + if (sink_to >= 0) + for (; sink_from <= sink_to; sink_from++) + folio_put(bvec_folio(&bvec[sink_from])); + kfree(bvec); return ret; } diff --git a/fs/netfs/buffered_write.c b/fs/netfs/buffered_write.c index 2cdb68e6b16f..ecf119b4f916 100644 --- a/fs/netfs/buffered_write.c +++ b/fs/netfs/buffered_write.c @@ -469,6 +469,7 @@ ssize_t netfs_buffered_write_iter_locked(struct kiocb *iocb, struct iov_iter *fr struct netfs_group *netfs_group) { struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); ssize_t ret; trace_netfs_write_iter(iocb, from); @@ -481,6 +482,14 @@ ssize_t netfs_buffered_write_iter_locked(struct kiocb *iocb, struct iov_iter *fr if (ret) return ret; + if (iocb->ki_pos > i_size_read(inode)) { + ret = netfs_clear_stale_pre_isize(inode, i_size_read(inode), + iocb->ki_pos, + iocb->ki_flags & IOCB_NOWAIT); + if (ret) + return ret; + } + return netfs_perform_write(iocb, from, netfs_group); } EXPORT_SYMBOL(netfs_buffered_write_iter_locked); diff --git a/fs/netfs/direct_write.c b/fs/netfs/direct_write.c index 2361277416c7..8be74706d85e 100644 --- a/fs/netfs/direct_write.c +++ b/fs/netfs/direct_write.c @@ -359,6 +359,15 @@ ssize_t netfs_unbuffered_write_iter(struct kiocb *iocb, struct iov_iter *from) ret = file_update_time(file); if (ret < 0) goto out; + + if (iocb->ki_pos > i_size_read(inode)) { + ret = netfs_clear_stale_pre_isize(inode, i_size_read(inode), + iocb->ki_pos, + iocb->ki_flags & IOCB_NOWAIT); + if (ret < 0) + goto out; + } + if (iocb->ki_flags & IOCB_NOWAIT) { /* We could block if there are any pages in the range. */ ret = -EAGAIN; diff --git a/fs/netfs/internal.h b/fs/netfs/internal.h index c79c8e69d60c..786af76da98c 100644 --- a/fs/netfs/internal.h +++ b/fs/netfs/internal.h @@ -80,6 +80,8 @@ ssize_t netfs_wait_for_write(struct netfs_io_request *rreq); void netfs_wait_for_paused_read(struct netfs_io_request *rreq); void netfs_wait_for_paused_write(struct netfs_io_request *rreq); void netfs_wait_for_put_ra_refs(struct netfs_io_request *rreq); +int netfs_clear_stale_pre_isize(struct inode *inode, uoff_t from, + uoff_t to, bool nowait); /* * objects.c diff --git a/fs/netfs/misc.c b/fs/netfs/misc.c index f5c1c463f4ff..523057390d6f 100644 --- a/fs/netfs/misc.c +++ b/fs/netfs/misc.c @@ -6,6 +6,7 @@ */ #include +#include #include "internal.h" /** @@ -582,3 +583,101 @@ void netfs_wait_for_put_ra_refs(struct netfs_io_request *rreq) trace_netfs_rreq(rreq, netfs_rreq_trace_waited_put_ra_refs); finish_wait(&rreq->waitq, &myself); } + +/** + * netfs_clear_stale_isize - Clear stale pagecache in a to-be-created hole + * @inode: The inode to act upon. + * @from: The base of the hole to be made. + * @to: The top of the hole to be made. + * @nowait: True to return -EAGAIN rather than block. + * @exclusive: True if the caller holds i_rwsem exclusively for the resize. + * + * Zero any data left in the pagecache within the [@from, @to) hole by a + * write through an mmap so that it isn't exposed as file content once the + * file is extended. Only the uptodate folio straddling @from can hold such + * data as pages wholly beyond the EOF can't be faulted in, so the zeroing + * is limited to that folio. The folio is zeroed rather than dropped so + * that a concurrent extending write can't lose data. + * + * If @exclusive is false, @from is re-read from i_size and used to clamp + * the zeroed range, for callers that may race with another writer also + * extending the file (eg. multiple buffered writes extending the same file + * under a shared i_rwsem). If @exclusive is true, for callers that hold + * i_rwsem exclusively across the whole resize and have already updated + * i_size to @to, staleness is decided from the folio's dirty state instead: + * since no genuine concurrent buffered writer can be racing, a lockless + * stat() adopting a server-confirmed size mid-resize has no data behind it + * and never dirties the folio, so it can't fool this check into skipping + * the zeroing the way it could fool the @exclusive false clamp. + * + * pagecache_isize_extended() can't be reused here: it is keyed on a + * sub-page block size (a no-op when the block size is >= PAGE_SIZE, as on + * network filesystems), runs after i_size is updated, can't honour + * @nowait, and doesn't wait for writeback. Keep the two in sync if either + * is changed. + * + * Return: 0 on success, or -EAGAIN if @nowait is set and the folio is + * mapped or under writeback and so can't be cleaned without blocking. + */ +static int netfs_clear_stale_isize(struct inode *inode, uoff_t from, + uoff_t to, bool nowait, bool exclusive) +{ + struct address_space *mapping = inode->i_mapping; + fgf_t fgp = FGP_LOCK; + struct folio *folio; + int ret; + + if (from >= to) + return 0; + + if (nowait) + fgp |= FGP_NOWAIT; + + folio = __filemap_get_folio(mapping, from >> PAGE_SHIFT, fgp, 0); + if (IS_ERR(folio)) + return PTR_ERR(folio) == -EAGAIN ? -EAGAIN : 0; + + ret = 0; + if (nowait && (folio_mapped(folio) || folio_test_writeback(folio))) { + ret = -EAGAIN; + goto out; + } + + folio_wait_writeback(folio); + + if (folio_mkclean(folio)) + folio_mark_dirty(folio); + + if (folio_test_uptodate(folio) && + (!exclusive || folio_test_dirty(folio))) { + uoff_t fpos = folio_pos(folio); + + if (!exclusive) + from = umax(from, i_size_read(inode)); + if (from < to && from < fpos + folio_size(folio)) { + size_t end = umin(to - fpos, folio_size(folio)); + size_t offset = from - fpos; + + folio_zero_segment(folio, offset, end); + } + } +out: + folio_unlock(folio); + folio_put(folio); + return ret; +} + +/* Clear stale pagecache before an extending buffered/DIO write. */ +int netfs_clear_stale_pre_isize(struct inode *inode, uoff_t from, + uoff_t to, bool nowait) +{ + return netfs_clear_stale_isize(inode, from, to, nowait, false); +} + +/* Clear stale pagecache when extending a file under an exclusive resize. */ +void netfs_clear_stale_post_isize(struct inode *inode, uoff_t from, + uoff_t to) +{ + netfs_clear_stale_isize(inode, from, to, false, true); +} +EXPORT_SYMBOL(netfs_clear_stale_post_isize); diff --git a/fs/netfs/objects.c b/fs/netfs/objects.c index 7f6a3e912602..ad549daa9c79 100644 --- a/fs/netfs/objects.c +++ b/fs/netfs/objects.c @@ -34,7 +34,7 @@ struct netfs_io_request *netfs_alloc_request(struct address_space *mapping, rreq = mempool_alloc(mempool, gfp); } else { - rreq = mempool->alloc(gfp, mempool->pool_data); + rreq = mempool_alloc_noreserve(mempool, gfp); if (!rreq) return ERR_PTR(-ENOMEM); } @@ -214,7 +214,7 @@ struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq struct kmem_cache *cache = mempool->pool_data; if (rreq->gfp == GFP_KERNEL) - subreq = mempool->alloc(rreq->gfp, mempool->pool_data); + subreq = mempool_alloc_noreserve(mempool, rreq->gfp); else subreq = mempool_alloc(mempool, rreq->gfp); if (!subreq) diff --git a/fs/netfs/read_collect.c b/fs/netfs/read_collect.c index 5cf22087d243..c5bfaf2c6f25 100644 --- a/fs/netfs/read_collect.c +++ b/fs/netfs/read_collect.c @@ -33,6 +33,22 @@ static void netfs_clear_unread(struct netfs_io_subrequest *subreq) __set_bit(NETFS_SREQ_HIT_EOF, &subreq->flags); } +static void netfs_clear_unread_dio(struct netfs_io_subrequest *subreq) +{ + uoff_t pos = subreq->start + subreq->transferred; + struct netfs_io_request *rreq = subreq->rreq; + size_t fill; + + if (pos >= rreq->i_size) + return; + + fill = min_t(uoff_t, rreq->i_size - pos, + subreq->len - subreq->transferred); + + netfs_reset_iter(subreq); + subreq->transferred += iov_iter_zero(fill, &subreq->io_iter); +} + /* * Cancel the copy-to-cache mark on a folio. */ @@ -311,6 +327,14 @@ static void netfs_collect_read_results(struct netfs_io_request *rreq) test_bit(NETFS_SREQ_HIT_EOF, &front->flags)) netfs_read_unlock_folios(rreq, ¬es); } else { + if (!(notes & HIT_PENDING) && + front->error == 0 && + transferred < front->len && + test_bit(NETFS_SREQ_CLEAR_TAIL, &front->flags)) { + netfs_clear_unread_dio(front); + transferred = front->transferred; + trace_netfs_sreq(front, netfs_sreq_trace_clear); + } stream->collected_to = front->start + transferred; rreq->collected_to = stream->collected_to; } @@ -435,6 +459,11 @@ static void netfs_rreq_assess_single(struct netfs_io_request *rreq) netfs_single_mark_inode_dirty(rreq->inode); } + /* To do DIO, the cache has to round the size up, so we need to undo + * the rounding. + */ + rreq->transferred = min(rreq->transferred, rreq->i_size); + if (rreq->iocb) { rreq->iocb->ki_pos += rreq->transferred; if (rreq->iocb->ki_complete) { diff --git a/fs/netfs/rolling_buffer.c b/fs/netfs/rolling_buffer.c index 424e77a9a109..d30d5ef6d86e 100644 --- a/fs/netfs/rolling_buffer.c +++ b/fs/netfs/rolling_buffer.c @@ -29,7 +29,7 @@ struct folio_queue *netfs_folioq_alloc(unsigned int rreq_id, gfp_t gfp, struct folio_queue *fq; if (gfp == GFP_KERNEL) - fq = netfs_folioq_pool.alloc(gfp, netfs_folioq_pool.pool_data); + fq = mempool_alloc_noreserve(&netfs_folioq_pool, gfp); else fq = mempool_alloc(&netfs_folioq_pool, gfp); if (fq) { diff --git a/fs/ntfs3/inode.c b/fs/ntfs3/inode.c index 56b4f6469a28..4ac26c80bd34 100644 --- a/fs/ntfs3/inode.c +++ b/fs/ntfs3/inode.c @@ -1866,10 +1866,10 @@ int ntfs_create_inode(struct mnt_idmap *idmap, struct inode *dir, goto out6; /* - * Call 'd_instantiate' after inode->i_op is set + * Call 'd_instantiate_new' after inode->i_op is set * but before finish_open. */ - d_instantiate(dentry, inode); + d_instantiate_new(dentry, inode); /* Set original time. inode times (i_ctime) may be changed in ntfs_init_acl. */ inode_set_atime_to_ts(inode, ni->i_crtime); @@ -1917,9 +1917,6 @@ int ntfs_create_inode(struct mnt_idmap *idmap, struct inode *dir, if (!fnd) ni_unlock(dir_ni); - if (!err) - unlock_new_inode(inode); - return err; } diff --git a/fs/ocfs2/namei.c b/fs/ocfs2/namei.c index e9c7774ccf91..58c6061ed983 100644 --- a/fs/ocfs2/namei.c +++ b/fs/ocfs2/namei.c @@ -336,13 +336,8 @@ static int ocfs2_mknod(struct mnt_idmap *idmap, goto leave; /* calculate meta data/clusters for setting security and acl xattr */ - status = ocfs2_calc_xattr_init(dir, mode, &si, &want_clusters, - &xattr_credits, &want_meta, - &acl_state); - if (status < 0) { - mlog_errno(status); - goto leave; - } + ocfs2_calc_xattr_init(dir, mode, &si, &want_clusters, &xattr_credits, + &want_meta, &acl_state); /* Reserve a cluster if creating an extent based directory. */ if (S_ISDIR(mode) && !ocfs2_supports_inline_data(osb)) { diff --git a/fs/ocfs2/xattr.c b/fs/ocfs2/xattr.c index 35bcbb0ff607..bfafe059bedf 100644 --- a/fs/ocfs2/xattr.c +++ b/fs/ocfs2/xattr.c @@ -635,12 +635,11 @@ int ocfs2_calc_security_init(struct inode *dir, return ret; } -int ocfs2_calc_xattr_init(struct inode *dir, umode_t mode, - struct ocfs2_security_xattr_info *si, - int *want_clusters, int *xattr_credits, - int *want_meta, struct ocfs2_acl_state *acl_state) +void ocfs2_calc_xattr_init(struct inode *dir, umode_t mode, + struct ocfs2_security_xattr_info *si, + int *want_clusters, int *xattr_credits, + int *want_meta, struct ocfs2_acl_state *acl_state) { - int ret = 0; struct ocfs2_super *osb = OCFS2_SB(dir->i_sb); int s_size = 0, a_size = 0, acl_len = 0, new_clusters; @@ -662,7 +661,7 @@ int ocfs2_calc_xattr_init(struct inode *dir, umode_t mode, } if (!(s_size + a_size)) - return ret; + return; /* * The max space of security xattr taken inline is @@ -728,8 +727,6 @@ int ocfs2_calc_xattr_init(struct inode *dir, umode_t mode, } } } - - return ret; } static int ocfs2_xattr_extend_allocation(struct inode *inode, diff --git a/fs/ocfs2/xattr.h b/fs/ocfs2/xattr.h index 5e18513277f1..887cc1a18b1a 100644 --- a/fs/ocfs2/xattr.h +++ b/fs/ocfs2/xattr.h @@ -59,10 +59,10 @@ int ocfs2_calc_security_init(struct inode *, int *, int *, struct ocfs2_alloc_context **); struct ocfs2_acl_state; -int ocfs2_calc_xattr_init(struct inode *dir, umode_t mode, - struct ocfs2_security_xattr_info *si, - int *want_clusters, int *xattr_credits, - int *want_meta, struct ocfs2_acl_state *acl_state); +void ocfs2_calc_xattr_init(struct inode *dir, umode_t mode, + struct ocfs2_security_xattr_info *si, + int *want_clusters, int *xattr_credits, + int *want_meta, struct ocfs2_acl_state *acl_state); /* * xattrs can live inside an inode, as part of an external xattr block, diff --git a/fs/overlayfs/overlayfs.h b/fs/overlayfs/overlayfs.h index e0d8c6152e9f..7f3558372c59 100644 --- a/fs/overlayfs/overlayfs.h +++ b/fs/overlayfs/overlayfs.h @@ -254,8 +254,10 @@ static inline struct dentry *ovl_do_mkdir(struct ovl_fs *ofs, { struct dentry *ret; + /* vfs_mkdir() drops @dentry on failure and may replace it on success */ + pr_debug("mkdir(%pd2, 0%o)\n", dentry, mode); ret = vfs_mkdir(ovl_upper_mnt_idmap(ofs), dir, dentry, mode, NULL); - pr_debug("mkdir(%pd2, 0%o) = %i\n", dentry, mode, PTR_ERR_OR_ZERO(ret)); + pr_debug("...mkdir = %i\n", PTR_ERR_OR_ZERO(ret)); return ret; } diff --git a/fs/smb/client/cached_dir.c b/fs/smb/client/cached_dir.c index 88d5e9a32f28..647fa26da4d2 100644 --- a/fs/smb/client/cached_dir.c +++ b/fs/smb/client/cached_dir.c @@ -8,6 +8,7 @@ #include #include "cifsglob.h" #include "cifsproto.h" +#include "../common/smb2status.h" #include "cifs_debug.h" #include "smb2proto.h" #include "cached_dir.h" @@ -323,25 +324,37 @@ int open_cached_dir(unsigned int xid, struct cifs_tcon *tcon, rc = compound_send_recv(xid, ses, server, flags, 2, rqst, resp_buftype, rsp_iov); - if (rc) { - if (rc == -EREMCHG) { - tcon->need_reconnect = true; - pr_warn_once("server share %s deleted\n", - tcon->tree_name); - } + if (rc == -EREMCHG) { + tcon->need_reconnect = true; + pr_warn_once("server share %s deleted\n", + tcon->tree_name); + } + + if (!rsp_iov[0].iov_base || rsp_iov[0].iov_len < sizeof(*o_rsp)) { + if (!rc) + rc = -EIO; goto oshr_free; } - cfid->is_open = true; - - spin_lock(&cfids->cfid_list_lock); o_rsp = (struct smb2_create_rsp *)rsp_iov[0].iov_base; + if (o_rsp->hdr.Status != STATUS_SUCCESS) { + if (!rc) + rc = -EIO; + goto oshr_free; + } + oparms.fid->persistent_fid = o_rsp->PersistentFileId; oparms.fid->volatile_fid = o_rsp->VolatileFileId; #ifdef CONFIG_CIFS_DEBUG2 oparms.fid->mid = le64_to_cpu(o_rsp->hdr.MessageId); #endif /* CIFS_DEBUG2 */ + cfid->is_open = true; + atomic_inc(&tcon->num_remote_opens); + if (rc) + goto oshr_free; + + spin_lock(&cfids->cfid_list_lock); if (o_rsp->OplockLevel != SMB2_OPLOCK_LEVEL_LEASE) { spin_unlock(&cfids->cfid_list_lock); @@ -408,7 +421,6 @@ int open_cached_dir(unsigned int xid, struct cifs_tcon *tcon, close_cached_dir(cfid); } else { *ret_cfid = cfid; - atomic_inc(&tcon->num_remote_opens); } kfree(utf16_path); diff --git a/fs/smb/client/cifsfs.c b/fs/smb/client/cifsfs.c index 7ecd70efdfea..98b610b2a114 100644 --- a/fs/smb/client/cifsfs.c +++ b/fs/smb/client/cifsfs.c @@ -1412,6 +1412,7 @@ static loff_t cifs_remap_file_range(struct file *src_file, loff_t off, * server could even support copy of range where source = target */ lock_two_nondirectories(target_inode, src_inode); + filemap_invalidate_lock(target_inode->i_mapping); if (len == 0) { loff_t src_size = i_size_read(src_inode); @@ -1467,9 +1468,15 @@ static loff_t cifs_remap_file_range(struct file *src_file, loff_t off, i_size = target_inode->i_size; spin_unlock(&target_inode->i_lock); - /* Discard all the folios that overlap the destination region. */ + /* + * Discard all the folios that overlap the destination region. Start at + * the old EOF when extending so the folio straddling it, which may hold + * data written past EOF through an mmap, is dropped too. + */ cifs_dbg(FYI, "about to discard pages %llx-%llx\n", fstart, fend); - truncate_inode_pages_range(&target_inode->i_data, fstart, fend); + truncate_inode_pages_range(&target_inode->i_data, + min(fstart, i_size), fend); + netfs_wait_for_outstanding_io(target_inode); fscache_invalidate(cifs_inode_cookie(target_inode), NULL, i_size, 0); @@ -1508,6 +1515,7 @@ static loff_t cifs_remap_file_range(struct file *src_file, loff_t off, if (rc) CIFS_I(target_inode)->time = 0; unlock: + filemap_invalidate_unlock(target_inode->i_mapping); /* although unlocking in the reverse order from locking is not strictly necessary here it is a little cleaner to be consistent */ unlock_two_nondirectories(src_inode, target_inode); @@ -1531,6 +1539,9 @@ ssize_t cifs_file_copychunk_range(unsigned int xid, struct cifs_tcon *target_tcon; ssize_t rc; + if (len == 0) + return 0; + cifs_dbg(FYI, "copychunk range\n"); if (!src_file->private_data || !dst_file->private_data) { @@ -1560,6 +1571,7 @@ ssize_t cifs_file_copychunk_range(unsigned int xid, * server could even support copy of range where source = target */ lock_two_nondirectories(target_inode, src_inode); + filemap_invalidate_lock(target_inode->i_mapping); cifs_dbg(FYI, "about to flush pages\n"); @@ -1581,10 +1593,28 @@ ssize_t cifs_file_copychunk_range(unsigned int xid, /* Flush and invalidate all the folios in the destination region. If * the copy was successful, then some of the flush is extra overhead, * but we need to allow for the copy failing in some way (eg. ENOSPC). + * + * Start at the old EOF when extending so the folio straddling it, which + * may hold data written past EOF through an mmap, is dropped too. */ - rc = filemap_invalidate_inode(target_inode, true, destoff, destoff + len - 1); - if (rc) - goto unlock; + if (target_inode->i_mapping->nrpages) { + loff_t fstart = min(destoff, i_size_read(target_inode)); + loff_t fend = destoff + len - 1; + + unmap_mapping_pages(target_inode->i_mapping, + fstart >> PAGE_SHIFT, + (fend >> PAGE_SHIFT) - + (fstart >> PAGE_SHIFT) + 1, + false); + rc = filemap_write_and_wait_range(target_inode->i_mapping, + fstart, fend); + if (rc) + goto unlock; + invalidate_inode_pages2_range(target_inode->i_mapping, + fstart >> PAGE_SHIFT, + fend >> PAGE_SHIFT); + } + netfs_wait_for_outstanding_io(target_inode); fscache_invalidate(cifs_inode_cookie(target_inode), NULL, i_size_read(target_inode), 0); @@ -1616,6 +1646,7 @@ ssize_t cifs_file_copychunk_range(unsigned int xid, CIFS_I(target_inode)->time = 0; unlock: + filemap_invalidate_unlock(target_inode->i_mapping); /* although unlocking in the reverse order from locking is not * strictly necessary here it is a little cleaner to be consistent */ diff --git a/fs/smb/client/cifsfs.h b/fs/smb/client/cifsfs.h index 0c85daa8386e..e3d820c9778b 100644 --- a/fs/smb/client/cifsfs.h +++ b/fs/smb/client/cifsfs.h @@ -146,8 +146,9 @@ ssize_t cifs_file_copychunk_range(unsigned int xid, struct file *src_file, unsigned int flags); long cifs_ioctl(struct file *filep, unsigned int command, unsigned long arg); -void cifs_setsize(struct inode *inode, loff_t offset); -void cifs_resize_file_locked(struct inode *inode, loff_t offset); +void cifs_setsize(struct inode *inode, loff_t old_size, loff_t offset); +void cifs_resize_file_locked(struct inode *inode, loff_t old_size, + loff_t offset); struct fs_context; struct smb3_fs_context; diff --git a/fs/smb/client/cifsglob.h b/fs/smb/client/cifsglob.h index 79e4e84f8985..5c5b76a9e9fe 100644 --- a/fs/smb/client/cifsglob.h +++ b/fs/smb/client/cifsglob.h @@ -1450,11 +1450,11 @@ struct cifsFileInfo { struct dentry *dentry; struct tcon_link *tlink; unsigned int f_flags; - bool invalidHandle:1; /* file closed via session abend */ - bool swapfile:1; - bool oplock_break_cancelled:1; - bool status_file_deleted:1; /* file has been deleted */ - bool offload:1; /* offload final part of _put to a wq */ + bool invalidHandle; /* file closed via session abend */ + bool swapfile; + bool oplock_break_cancelled; + bool status_file_deleted; /* file has been deleted */ + bool offload; /* offload final part of _put to a wq */ __u16 oplock_epoch; /* epoch from the lease break */ __u32 oplock_level; /* oplock/lease level from the lease break */ int count; diff --git a/fs/smb/client/dfs_cache.c b/fs/smb/client/dfs_cache.c index 29dfd7595941..26956e33037e 100644 --- a/fs/smb/client/dfs_cache.c +++ b/fs/smb/client/dfs_cache.c @@ -798,13 +798,13 @@ static int get_targets(struct cache_entry *ce, struct dfs_cache_tgt_list *tl) INIT_LIST_HEAD(head); list_for_each_entry(t, &ce->tlist, list) { - it = kzalloc_obj(*it, GFP_ATOMIC); + it = kzalloc_obj(*it, GFP_KERNEL); if (!it) { rc = -ENOMEM; goto err_free_it; } - it->it_name = kstrdup(t->name, GFP_ATOMIC); + it->it_name = kstrdup(t->name, GFP_KERNEL); if (!it->it_name) { kfree(it); rc = -ENOMEM; diff --git a/fs/smb/client/dir.c b/fs/smb/client/dir.c index 6fa6d48fdfd3..1a56fa4d0e89 100644 --- a/fs/smb/client/dir.c +++ b/fs/smb/client/dir.c @@ -199,7 +199,7 @@ static int __cifs_do_create(struct inode *dir, struct dentry *direntry, struct tcon_link *tlink, unsigned int oflags, umode_t mode, __u32 *oplock, struct cifs_fid *fid, struct cifs_open_info_data *buf, - struct inode **inode) + struct inode **inode, bool *opened) { int rc = -ENOENT; int create_options = CREATE_NOT_DIR; @@ -216,6 +216,7 @@ static int __cifs_do_create(struct inode *dir, struct dentry *direntry, __le32 lease_flags = 0; *inode = NULL; + *opened = false; *oplock = 0; if (tcon->ses->server->oplocks) *oplock = REQ_OPLOCK; @@ -232,6 +233,7 @@ static int __cifs_do_create(struct inode *dir, struct dentry *direntry, oflags, oplock, &fid->netfid, xid); switch (rc) { case 0: + *opened = true; if (newinode == NULL) { /* query inode info */ goto cifs_create_get_file_info; @@ -253,11 +255,9 @@ static int __cifs_do_create(struct inode *dir, struct dentry *direntry, /* * The server may allow us to open things like * FIFOs, but the client isn't set up to deal - * with that. If it's not a regular file, just - * close it and proceed as if it were a normal - * lookup. + * with that. Keep the handle until the caller + * can finish the lookup. */ - CIFSSMBClose(xid, tcon, fid->netfid); goto cifs_create_get_file_info; } /* success, no need to query */ @@ -384,6 +384,7 @@ static int __cifs_do_create(struct inode *dir, struct dentry *direntry, } return rc; } + *opened = true; if (rdwr_for_fscache == 2) cifs_invalidate_cache(dir, FSCACHE_INVAL_DIO_WRITE); @@ -475,7 +476,7 @@ static int __cifs_do_create(struct inode *dir, struct dentry *direntry, return rc; out_err: - if (server->ops->close) + if (*opened && server->ops->close) server->ops->close(xid, tcon, fid); if (newinode) iput(newinode); @@ -487,7 +488,7 @@ static int cifs_do_create(struct inode *dir, struct dentry *direntry, unsigned int oflags, umode_t mode, __u32 *oplock, struct cifs_fid *fid, struct cifs_open_info_data *buf, - struct inode **inode) + struct inode **inode, bool *opened) { void *page = alloc_dentry_path(); const char *full_path; @@ -496,10 +497,11 @@ static int cifs_do_create(struct inode *dir, struct dentry *direntry, full_path = build_path_from_dentry(direntry, page); if (IS_ERR(full_path)) { rc = PTR_ERR(full_path); + *opened = false; } else { rc = __cifs_do_create(dir, direntry, full_path, xid, tlink, oflags, mode, oplock, - fid, buf, inode); + fid, buf, inode, opened); } free_dentry_path(page); return rc; @@ -529,6 +531,8 @@ int cifs_atomic_open(struct inode *dir, struct dentry *direntry, struct inode *inode; unsigned int xid; __u32 oplock; + bool is_regular; + bool opened; int rc; if (unlikely(cifs_forced_shutdown(cifs_sb))) @@ -581,12 +585,26 @@ int cifs_atomic_open(struct inode *dir, struct dentry *direntry, cifs_add_pending_open(&fid, tlink, &open); rc = cifs_do_create(dir, direntry, xid, tlink, oflags, mode, - &oplock, &fid, &buf, &inode); + &oplock, &fid, &buf, &inode, &opened); if (rc) { cifs_del_pending_open(&open); goto out; } + is_regular = S_ISREG(inode->i_mode); + if (!is_regular || !opened) { + if (opened && server->ops->close) + server->ops->close(xid, tcon, &fid); + cifs_del_pending_open(&open); + if (S_ISLNK(inode->i_mode) && + (oflags & (O_NOFOLLOW | __O_REGULAR)) == + (O_NOFOLLOW | __O_REGULAR) && !(oflags & O_EXCL)) { + iput(inode); + rc = -ELOOP; + goto out; + } + } + if (d_in_lookup(direntry)) { alias = d_splice_alias(inode, direntry); if (!IS_ERR_OR_NULL(alias)) @@ -595,9 +613,15 @@ int cifs_atomic_open(struct inode *dir, struct dentry *direntry, d_instantiate(direntry, inode); } - if ((oflags & (O_CREAT | O_EXCL)) == (O_CREAT | O_EXCL)) + if (is_regular && opened && + (oflags & (O_CREAT | O_EXCL)) == (O_CREAT | O_EXCL)) file->f_mode |= FMODE_CREATED; + if (!is_regular || !opened) { + rc = finish_no_open(file, NULL); + goto out; + } + rc = finish_open(file, direntry, generic_file_open); if (rc) { if (server->ops->close) @@ -660,6 +684,7 @@ int cifs_create(struct mnt_idmap *idmap, struct inode *dir, struct inode *inode; struct cifs_fid fid; __u32 oplock; + bool opened; struct cifs_open_info_data buf = {}; cifs_dbg(FYI, "cifs_create parent inode = 0x%p name is: %pd and dentry = 0x%p\n", @@ -682,10 +707,10 @@ int cifs_create(struct mnt_idmap *idmap, struct inode *dir, server->ops->new_lease_key(&fid); rc = cifs_do_create(dir, direntry, xid, tlink, oflags, - mode, &oplock, &fid, &buf, &inode); + mode, &oplock, &fid, &buf, &inode, &opened); if (!rc) { d_instantiate(direntry, inode); - if (server->ops->close) + if (opened && server->ops->close) server->ops->close(xid, tcon, &fid); } @@ -1078,6 +1103,7 @@ int cifs_tmpfile(struct mnt_idmap *idmap, struct inode *dir, struct inode *inode; unsigned int xid; __u32 oplock; + bool opened; int namelen; int rc; @@ -1116,7 +1142,7 @@ int cifs_tmpfile(struct mnt_idmap *idmap, struct inode *dir, namelen = scnprintf(name, namesize, CIFS_TMPNAME_PREFIX "%x", atomic_inc_return(&cifs_tmpcounter)); rc = __cifs_do_create(dir, dentry, path, xid, tlink, oflags, - mode, &oplock, &fid, NULL, &inode); + mode, &oplock, &fid, NULL, &inode, &opened); if (!rc) { rc = d_mark_tmpfile_name(file, &QSTR_LEN(name, namelen)); if (rc) { diff --git a/fs/smb/client/file.c b/fs/smb/client/file.c index 0d428517f454..11355d6bf89d 100644 --- a/fs/smb/client/file.c +++ b/fs/smb/client/file.c @@ -1012,11 +1012,14 @@ static int cifs_do_truncate(const unsigned int xid, struct dentry *dentry) } mapping_set_error(inode->i_mapping, rc); + netfs_wait_for_outstanding_io(inode); + cfile = find_writable_file(cinode, FIND_FSUID_ONLY); rc = cifs_file_flush(xid, inode, cfile); if (!rc) { if (cfile) { struct netfs_inode *ictx = netfs_inode(inode); + loff_t old_size = i_size_read(inode); tcon = tlink_tcon(cfile->tlink); server = tcon->ses->server; @@ -1025,7 +1028,7 @@ static int cifs_do_truncate(const unsigned int xid, struct dentry *dentry) cfile, 0, false); if (!rc) { netfs_resize_file(&cinode->netfs, 0, true); - cifs_setsize(inode, 0); + cifs_setsize(inode, old_size, 0); cifs_invalidate_cache(inode, 0); } netfs_wb_end(ictx); diff --git a/fs/smb/client/inode.c b/fs/smb/client/inode.c index 1fe0ef0a95db..f29555f3c7d0 100644 --- a/fs/smb/client/inode.c +++ b/fs/smb/client/inode.c @@ -88,6 +88,8 @@ static void cifs_set_ops(struct inode *inode) else inode->i_data.a_ops = &cifs_addr_ops; mapping_set_large_folios(inode->i_mapping); + if (tcon->ses->server->sign) + mapping_set_stable_writes(inode->i_mapping); break; case S_IFDIR: if (IS_AUTOMOUNT(inode)) { @@ -3053,15 +3055,12 @@ int cifs_fiemap(struct inode *inode, struct fiemap_extent_info *fei, u64 start, return -EOPNOTSUPP; } -void cifs_setsize(struct inode *inode, loff_t offset) +void cifs_setsize(struct inode *inode, loff_t old_size, loff_t offset) { - loff_t old_size; u64 blocks = CIFS_INO_BLOCKS(offset); spin_lock(&inode->i_lock); - old_size = i_size_read(inode); i_size_write(inode, offset); - /* * Extending EOF does not allocate the intervening range. Only clamp * i_blocks on shrink; allocation growth comes from writes or from the @@ -3071,20 +3070,28 @@ void cifs_setsize(struct inode *inode, loff_t offset) inode->i_blocks = blocks; spin_unlock(&inode->i_lock); inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + + /* + * Zero the tail of the folio straddling the old EOF so data dirtied + * past EOF through an mmap isn't exposed. truncate_pagecache() then + * drops any pagecache beyond the new EOF, as in truncate_setsize(). + */ if (offset > old_size) - pagecache_isize_extended(inode, old_size, offset); + netfs_clear_stale_post_isize(inode, old_size, offset); + truncate_pagecache(inode, offset); netfs_wait_for_outstanding_io(inode); } -void cifs_resize_file_locked(struct inode *inode, loff_t offset) +void cifs_resize_file_locked(struct inode *inode, loff_t old_size, + loff_t offset) { struct fscache_cookie *cookie = cifs_inode_cookie(inode); lockdep_assert_held_write(&inode->i_rwsem); netfs_resize_file(netfs_inode(inode), offset, true); - cifs_setsize(inode, offset); + cifs_setsize(inode, old_size, offset); if (!cookie) return; @@ -3101,6 +3108,7 @@ int cifs_file_set_size(const unsigned int xid, struct dentry *dentry, struct inode *inode = d_inode(dentry); struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); struct cifsInodeInfo *cifsInode = CIFS_I(inode); + loff_t old_size = i_size_read(inode); struct tcon_link *tlink = NULL; struct cifs_tcon *tcon = NULL; struct TCP_Server_Info *server; @@ -3159,7 +3167,7 @@ int cifs_file_set_size(const unsigned int xid, struct dentry *dentry, set_size_out: if (rc == 0) - cifs_resize_file_locked(inode, size); + cifs_resize_file_locked(inode, old_size, size); return rc; } diff --git a/fs/smb/client/smb2inode.c b/fs/smb/client/smb2inode.c index 13fe8e3b48f3..f46a62eae659 100644 --- a/fs/smb/client/smb2inode.c +++ b/fs/smb/client/smb2inode.c @@ -598,8 +598,10 @@ static int smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, /* smb2_parse_contexts() fills idata->fi.IndexNumber */ rc = smb2_parse_contexts(server, &rsp_iov[0], &oparms->fid->epoch, oparms->fid->lease_key, &oplock, &idata->fi, NULL); - if (rc) + if (rc) { cifs_dbg(VFS, "rc: %d parsing context of compound op\n", rc); + tmp_rc = rc; + } } for (i = 0; i < num_cmds; i++) { @@ -1121,7 +1123,7 @@ smb2_unlink(const unsigned int xid, struct cifs_tcon *tcon, const char *name, struct kvec close_iov; int resp_buftype[2]; struct cifs_fid fid; - int flags = 0; + int flags = CIFS_CP_CREATE_CLOSE_OP; __u8 oplock; int rc; diff --git a/fs/smb/client/smb2misc.c b/fs/smb/client/smb2misc.c index 0cfe60ae42c3..5e5cf92d1eb3 100644 --- a/fs/smb/client/smb2misc.c +++ b/fs/smb/client/smb2misc.c @@ -834,7 +834,8 @@ smb2_cancelled_close_fid(struct work_struct *work) */ static int __smb2_handle_cancelled_cmd(struct cifs_tcon *tcon, __u16 cmd, __u64 mid, - __u64 persistent_fid, __u64 volatile_fid) + __u64 persistent_fid, __u64 volatile_fid, + bool account_remote_open) { struct close_cancelled_open *cancelled; @@ -848,6 +849,8 @@ __smb2_handle_cancelled_cmd(struct cifs_tcon *tcon, __u16 cmd, __u64 mid, cancelled->cmd = cmd; cancelled->mid = mid; INIT_WORK(&cancelled->work, smb2_cancelled_close_fid); + if (account_remote_open) + atomic_inc(&tcon->num_remote_opens); WARN_ON(queue_work(cifsiod_wq, &cancelled->work) == false); return 0; @@ -884,7 +887,7 @@ smb2_handle_cancelled_close(struct cifs_tcon *tcon, __u64 persistent_fid, spin_unlock(&tcon->tc_lock); rc = __smb2_handle_cancelled_cmd(tcon, SMB2_CLOSE_HE, 0, - persistent_fid, volatile_fid); + persistent_fid, volatile_fid, false); if (rc) cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_cancelled_close); @@ -912,7 +915,7 @@ smb2_handle_cancelled_mid(struct mid_q_entry *mid, struct TCP_Server_Info *serve le16_to_cpu(hdr->Command), le64_to_cpu(hdr->MessageId), rsp->PersistentFileId, - rsp->VolatileFileId); + rsp->VolatileFileId, true); if (rc) cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_cancelled_mid); diff --git a/fs/smb/client/smb2ops.c b/fs/smb/client/smb2ops.c index 3464470d3297..583ab4c4d773 100644 --- a/fs/smb/client/smb2ops.c +++ b/fs/smb/client/smb2ops.c @@ -2287,6 +2287,7 @@ smb2_duplicate_extents(const unsigned int xid, struct duplicate_extents_to_file dup_ext_buf; struct timespec64 ts; struct cifs_tcon *tcon = tlink_tcon(trgtfile->tlink); + loff_t old_size; u64 asize; /* server fileays advertise duplicate extent support with this flag */ @@ -2305,11 +2306,12 @@ smb2_duplicate_extents(const unsigned int xid, trgtfile->fid.volatile_fid, tcon->tid, tcon->ses->Suid, src_off, dest_off, len); inode = d_inode(trgtfile->dentry); - if (i_size_read(inode) < dest_off + len) { + old_size = i_size_read(inode); + if (old_size < dest_off + len) { rc = smb2_set_file_size(xid, tcon, trgtfile, dest_off + len, false); if (rc) goto duplicate_extents_out; - cifs_resize_file_locked(inode, dest_off + len); + cifs_resize_file_locked(inode, old_size, dest_off + len); } rc = SMB2_ioctl(xid, tcon, trgtfile->fid.persistent_fid, trgtfile->fid.volatile_fid, @@ -3520,6 +3522,21 @@ static long smb3_zero_data(struct file *file, struct cifs_tcon *tcon, 0, NULL, NULL); } +static long query_server_eof(const unsigned int xid, + struct cifs_tcon *tcon, + struct cifsFileInfo *cfile, + unsigned long long *eof) +{ + struct smb2_file_all_info file_inf = {}; + long rc; + + rc = SMB2_query_info(xid, tcon, cfile->fid.persistent_fid, + cfile->fid.volatile_fid, &file_inf); + if (!rc) + *eof = le64_to_cpu(file_inf.EndOfFile); + return rc; +} + static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon, unsigned long long offset, unsigned long long len, bool keep_size) @@ -3547,25 +3564,32 @@ static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon, filemap_invalidate_lock(inode->i_mapping); netfs_read_sizes(inode, &i_size, &remote_i_size, &zero_point); - if (offset + len >= remote_i_size && offset < i_size) { - unsigned long long top = umin(offset + len, i_size); - rc = filemap_write_and_wait_range(inode->i_mapping, offset, top - 1); - if (rc < 0) - goto zero_range_exit; - } + rc = filemap_write_and_wait_range(inode->i_mapping, offset, + offset + len - 1); + if (rc < 0) + goto zero_range_exit; /* * We zero the range through ioctl, so we need remove the page caches * first, otherwise the data may be inconsistent with the server. + * + * Start at the old EOF when extending so the folio straddling it, which + * may hold data written past EOF through an mmap, is dropped too. */ - truncate_pagecache_range(inode, offset, offset + len - 1); + truncate_pagecache_range(inode, min(offset, i_size), offset + len - 1); netfs_wait_for_outstanding_io(inode); - /* if file not oplocked can't be sure whether asking to extend size */ - rc = -EOPNOTSUPP; - if (keep_size == false && !CIFS_CACHE_READ(cifsi)) - goto zero_range_exit; + if (!keep_size && !CIFS_CACHE_READ(cifsi)) { + rc = query_server_eof(xid, tcon, cfile, &remote_i_size); + if (rc) + goto zero_range_exit; + i_size = max(i_size, remote_i_size); + if (i_size < new_size) { + rc = -EOPNOTSUPP; + goto zero_range_exit; + } + } fscache_invalidate(cifs_inode_cookie(inode), NULL, i_size_read(inode), 0); @@ -3577,7 +3601,7 @@ static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon, /* * do we also need to change the size of the file? */ - if (keep_size == false && (unsigned long long)i_size_read(inode) < new_size) { + if (!keep_size && umax(i_size, i_size_read(inode)) < new_size) { rc = SMB2_set_eof(xid, tcon, cfile->fid.persistent_fid, cfile->fid.volatile_fid, cfile->pid, new_size); if (rc >= 0) { @@ -3723,7 +3747,8 @@ static int smb3_simple_fallocate_write_range(unsigned int xid, static int smb3_simple_fallocate_range(unsigned int xid, struct cifs_tcon *tcon, struct cifsFileInfo *cfile, - loff_t off, loff_t len) + loff_t off, loff_t len, + loff_t old_eof) { struct file_allocated_range_buffer in_data, *out_data = NULL, *tmp_data; struct inode *inode = d_inode(cfile->dentry); @@ -3739,12 +3764,30 @@ static int smb3_simple_fallocate_range(unsigned int xid, goto out; } - if (off >= i_size_read(inode)) { + if (off >= old_eof) { rc = smb3_simple_fallocate_write_range(xid, tcon, cfile, off, len, buf); goto out; } + filemap_invalidate_lock(inode->i_mapping); + + /* + * Flush and commit the data to the server, otherwise + * FSCTL_QUERY_ALLOCATED_RANGES might report recently written data as + * unallocated holes on Windows Servers, and the loop below would + * then zero-fill them and corrupt the file. + */ + rc = filemap_write_and_wait_range(inode->i_mapping, off, + off + len - 1); + if (rc) + goto out_unlock; + netfs_wait_for_outstanding_io(inode); + rc = SMB2_flush(xid, tcon, cfile->fid.persistent_fid, + cfile->fid.volatile_fid); + if (rc) + goto out_unlock; + in_data.file_offset = cpu_to_le64(off); in_data.length = cpu_to_le64(len); rc = SMB2_ioctl(xid, tcon, cfile->fid.persistent_fid, @@ -3754,7 +3797,7 @@ static int smb3_simple_fallocate_range(unsigned int xid, 1024 * sizeof(struct file_allocated_range_buffer), (char **)&out_data, &out_data_len); if (rc) - goto out; + goto out_unlock; tmp_data = out_data; while (len) { @@ -3764,12 +3807,12 @@ static int smb3_simple_fallocate_range(unsigned int xid, if (out_data_len == 0) { rc = smb3_simple_fallocate_write_range(xid, tcon, cfile, off, len, buf); - goto out; + goto out_unlock; } if (out_data_len < sizeof(struct file_allocated_range_buffer)) { rc = -EINVAL; - goto out; + goto out_unlock; } range_start = le64_to_cpu(tmp_data->file_offset); @@ -3777,7 +3820,7 @@ static int smb3_simple_fallocate_range(unsigned int xid, if (check_add_overflow(range_start, range_len, &range_end) || range_end > S64_MAX) { rc = -EINVAL; - goto out; + goto out_unlock; } if (off < range_start) { @@ -3792,11 +3835,11 @@ static int smb3_simple_fallocate_range(unsigned int xid, rc = smb3_simple_fallocate_write_range(xid, tcon, cfile, off, l, buf); if (rc) - goto out; + goto out_unlock; off = off + l; len = len - l; if (len == 0) - goto out; + goto out_unlock; } /* * We are at a section of allocated data, just skip forward @@ -3815,6 +3858,8 @@ static int smb3_simple_fallocate_range(unsigned int xid, out_data_len -= sizeof(struct file_allocated_range_buffer); } + out_unlock: + filemap_invalidate_unlock(inode->i_mapping); out: kfree(out_data); kvfree(buf); @@ -3830,7 +3875,7 @@ static long smb3_simple_falloc(struct file *file, struct cifs_tcon *tcon, struct cifsFileInfo *cfile = file->private_data; long rc = -EOPNOTSUPP; unsigned int xid; - loff_t old_eof, new_eof; + loff_t old_eof, new_eof, local_eof; struct smb2_file_all_info file_inf; u64 asize; int qrc; @@ -3839,18 +3884,37 @@ static long smb3_simple_falloc(struct file *file, struct cifs_tcon *tcon, inode = d_inode(cfile->dentry); cifsi = CIFS_I(inode); - old_eof = i_size_read(inode); + old_eof = local_eof = i_size_read(inode); trace_smb3_falloc_enter(xid, cfile->fid.persistent_fid, tcon->tid, tcon->ses->Suid, off, len); - /* if file not oplocked can't be sure whether asking to extend size */ - if (!CIFS_CACHE_READ(cifsi)) - if (!keep_size) { + + if (!keep_size && !CIFS_CACHE_READ(cifsi)) { + unsigned long long server_eof; + + rc = filemap_write_and_wait(inode->i_mapping); + if (rc) { trace_smb3_falloc_err(xid, cfile->fid.persistent_fid, tcon->tid, tcon->ses->Suid, off, len, rc); free_xid(xid); return rc; } + netfs_wait_for_outstanding_io(inode); + + rc = query_server_eof(xid, tcon, cfile, &server_eof); + if (rc) { + trace_smb3_falloc_err(xid, cfile->fid.persistent_fid, + tcon->tid, tcon->ses->Suid, off, len, rc); + free_xid(xid); + return rc; + } + /* + * Only use the larger EOF to decide whether we're extending. + * The pagecache zeroing below must still key off the local + * i_size, so keep local_eof for that. + */ + old_eof = max_t(loff_t, old_eof, server_eof); + } /* * Extending the file @@ -3874,7 +3938,7 @@ static long smb3_simple_falloc(struct file *file, struct cifs_tcon *tcon, } rc = smb3_simple_fallocate_range(xid, tcon, cfile, - off, len); + off, len, old_eof); if (rc) { spin_lock(&inode->i_lock); cifsi->time = 0; @@ -3883,7 +3947,7 @@ static long smb3_simple_falloc(struct file *file, struct cifs_tcon *tcon, } new_eof = off + len; - cifs_resize_file_locked(inode, new_eof); + cifs_resize_file_locked(inode, local_eof, new_eof); qrc = SMB2_query_info(xid, tcon, cfile->fid.persistent_fid, @@ -3931,7 +3995,7 @@ static long smb3_simple_falloc(struct file *file, struct cifs_tcon *tcon, if (rc) goto out; - cifs_resize_file_locked(inode, new_eof); + cifs_resize_file_locked(inode, local_eof, new_eof); qrc = SMB2_query_info(xid, tcon, cfile->fid.persistent_fid, @@ -3976,7 +4040,7 @@ static long smb3_simple_falloc(struct file *file, struct cifs_tcon *tcon, } } - if ((keep_size == true) || (i_size_read(inode) >= off + len)) { + if (keep_size || old_eof >= off + len) { /* * At this point, we are trying to fallocate an internal * regions of a sparse file. Since smb2 does not have a @@ -3993,7 +4057,7 @@ static long smb3_simple_falloc(struct file *file, struct cifs_tcon *tcon, */ if (len <= 1024 * 1024) { rc = smb3_simple_fallocate_range(xid, tcon, cfile, - off, len); + off, len, old_eof); goto out; } @@ -4005,7 +4069,7 @@ static long smb3_simple_falloc(struct file *file, struct cifs_tcon *tcon, * ie potentially making a few extra pages at the beginning * or end of the file non-sparse via set_sparse is harmless. */ - if ((off > 8192) || (off + len + 8192 < i_size_read(inode))) { + if (off > 8192 || off + len + 8192 < old_eof) { rc = -EOPNOTSUPP; goto out; } @@ -4572,25 +4636,37 @@ smb3_create_lease_buf(u8 *lease_key, u8 oplock, u8 *parent_lease_key, __le32 fla static __u8 smb2_parse_lease_buf(void *buf, __u16 *epoch, char *lease_key) { - struct create_lease *lc = (struct create_lease *)buf; + struct create_context *cc = buf; + struct lease_context lc; *epoch = 0; /* not used */ - if (lc->lcontext.LeaseFlags & SMB2_LEASE_FLAG_BREAK_IN_PROGRESS_LE) + if (le32_to_cpu(cc->DataLength) != sizeof(lc)) + return 0; + + memcpy(&lc, (u8 *)cc + le16_to_cpu(cc->DataOffset), sizeof(lc)); + if (lc.LeaseFlags & SMB2_LEASE_FLAG_BREAK_IN_PROGRESS_LE) return SMB2_OPLOCK_LEVEL_NOCHANGE; - return le32_to_cpu(lc->lcontext.LeaseState); + return le32_to_cpu(lc.LeaseState); } static __u8 smb3_parse_lease_buf(void *buf, __u16 *epoch, char *lease_key) { - struct create_lease_v2 *lc = (struct create_lease_v2 *)buf; + struct create_context *cc = buf; + struct lease_context_v2 lc; - *epoch = le16_to_cpu(lc->lcontext.Epoch); - if (lc->lcontext.LeaseFlags & SMB2_LEASE_FLAG_BREAK_IN_PROGRESS_LE) + if (le32_to_cpu(cc->DataLength) != sizeof(lc)) { + *epoch = 0; + return 0; + } + + memcpy(&lc, (u8 *)cc + le16_to_cpu(cc->DataOffset), sizeof(lc)); + *epoch = le16_to_cpu(lc.Epoch); + if (lc.LeaseFlags & SMB2_LEASE_FLAG_BREAK_IN_PROGRESS_LE) return SMB2_OPLOCK_LEVEL_NOCHANGE; if (lease_key) - memcpy(lease_key, &lc->lcontext.LeaseKey, SMB2_LEASE_KEY_SIZE); - return le32_to_cpu(lc->lcontext.LeaseState); + memcpy(lease_key, lc.LeaseKey, SMB2_LEASE_KEY_SIZE); + return le32_to_cpu(lc.LeaseState); } static unsigned int diff --git a/fs/smb/client/smb2pdu.c b/fs/smb/client/smb2pdu.c index 880ce12f50c4..cfe3c4b4abe7 100644 --- a/fs/smb/client/smb2pdu.c +++ b/fs/smb/client/smb2pdu.c @@ -2379,23 +2379,32 @@ create_reconnect_durable_buf(struct cifs_fid *fid) static void parse_query_id_ctxt(struct create_context *cc, struct smb2_file_all_info *buf) { - struct create_disk_id_rsp *pdisk_id = (struct create_disk_id_rsp *)cc; + u16 doff = le16_to_cpu(cc->DataOffset); + u32 dlen = le32_to_cpu(cc->DataLength); + u8 *beg; - cifs_dbg(FYI, "parse query id context 0x%llx 0x%llx\n", - pdisk_id->DiskFileId, pdisk_id->VolumeId); - buf->IndexNumber = pdisk_id->DiskFileId; + if (dlen < sizeof(__le64)) + return; + + beg = (u8 *)cc + doff; + memcpy(&buf->IndexNumber, beg, sizeof(__le64)); + cifs_dbg(FYI, "parse query id context 0x%llx\n", + le64_to_cpu(buf->IndexNumber)); } static void parse_posix_ctxt(struct create_context *cc, struct smb2_file_all_info *info, struct create_posix_rsp *posix) { - int sid_len; u8 *beg = (u8 *)cc + le16_to_cpu(cc->DataOffset); - u8 *end = beg + le32_to_cpu(cc->DataLength); + u32 dlen = le32_to_cpu(cc->DataLength); + u8 *end = beg + dlen; + int sid_len; u8 *sid; memset(posix, 0, sizeof(*posix)); + if (dlen < 3 * sizeof(__le32)) + return; posix->nlink = get_unaligned_le32(beg); posix->reparse_tag = get_unaligned_le32(beg + 4); @@ -2431,6 +2440,7 @@ int smb2_parse_contexts(struct TCP_Server_Info *server, struct smb2_create_rsp *rsp = rsp_iov->iov_base; struct create_context *cc; size_t rem, off, len; + size_t cc_len; size_t doff, dlen; size_t noff, nlen; char *name; @@ -2453,29 +2463,41 @@ int smb2_parse_contexts(struct TCP_Server_Info *server, buf->IndexNumber = 0; while (rem >= sizeof(*cc)) { + off = le32_to_cpu(cc->Next); + if (off) { + if ((off & 0x7) || off >= rem || off < sizeof(*cc)) + return -EINVAL; + cc_len = off; + } else { + cc_len = rem; + } + doff = le16_to_cpu(cc->DataOffset); dlen = le32_to_cpu(cc->DataLength); - if (check_add_overflow(doff, dlen, &len) || len > rem) + if (doff < sizeof(*cc) || + check_add_overflow(doff, dlen, &len) || len > cc_len) return -EINVAL; noff = le16_to_cpu(cc->NameOffset); nlen = le16_to_cpu(cc->NameLength); - if (noff + nlen > doff) + if (noff < sizeof(*cc) || + check_add_overflow(noff, nlen, &len) || len > cc_len || + (dlen && len > doff)) return -EINVAL; name = (char *)cc + noff; switch (nlen) { case 4: - if (!strncmp(name, SMB2_CREATE_REQUEST_LEASE, 4)) { + if (dlen && !strncmp(name, SMB2_CREATE_REQUEST_LEASE, 4)) { *oplock = server->ops->parse_lease_buf(cc, epoch, lease_key); - } else if (buf && + } else if (dlen && buf && !strncmp(name, SMB2_CREATE_QUERY_ON_DISK_ID, 4)) { parse_query_id_ctxt(cc, buf); } break; case 16: - if (posix && !memcmp(name, smb3_create_tag_posix, 16)) + if (dlen && posix && !memcmp(name, smb3_create_tag_posix, 16)) parse_posix_ctxt(cc, buf, posix); break; default: @@ -2487,13 +2509,18 @@ int smb2_parse_contexts(struct TCP_Server_Info *server, } off = le32_to_cpu(cc->Next); - if (!off) + if (!off) { + rem = 0; break; + } if (check_sub_overflow(rem, off, &rem)) return -EINVAL; cc = (struct create_context *)((u8 *)cc + off); } + if (rem) + return -EINVAL; + if (rsp->OplockLevel != SMB2_OPLOCK_LEVEL_LEASE) *oplock = rsp->OplockLevel; @@ -3389,6 +3416,9 @@ SMB2_open(const unsigned int xid, struct cifs_open_parms *oparms, __le16 *path, rc = smb2_parse_contexts(server, &rsp_iov, &oparms->fid->epoch, oparms->fid->lease_key, oplock, file_info, posix); + if (rc) + SMB2_close(xid, tcon, oparms->fid->persistent_fid, + oparms->fid->volatile_fid); trace_smb3_open_done(xid, rsp->PersistentFileId, tcon->tid, ses->Suid, oparms->create_options, oparms->desired_access, @@ -4766,14 +4796,20 @@ smb2_readv_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid) rdata->got_bytes); if (rdata->result == -ENODATA) { - __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags); rdata->result = 0; + if (rdata->subreq.start + rdata->subreq.transferred >= i_size_read(inode)) + __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags); + else + __set_bit(NETFS_SREQ_CLEAR_TAIL, &rdata->subreq.flags); } else { size_t trans = rdata->subreq.transferred + rdata->got_bytes; if (trans < rdata->subreq.len && rdata->subreq.start + trans >= netfs_read_remote_i_size(inode)) { - __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags); rdata->result = 0; + if (rdata->subreq.start + trans >= i_size_read(inode)) + __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags); + else + __set_bit(NETFS_SREQ_CLEAR_TAIL, &rdata->subreq.flags); } if (rdata->got_bytes) __set_bit(NETFS_SREQ_MADE_PROGRESS, &rdata->subreq.flags); diff --git a/fs/smb/client/smb2pdu.h b/fs/smb/client/smb2pdu.h index ab6c667bebc0..b37a1e3941a1 100644 --- a/fs/smb/client/smb2pdu.h +++ b/fs/smb/client/smb2pdu.h @@ -224,8 +224,7 @@ struct smb2_file_id_extd_directory_info { extern char smb2_padding[7]; /* - * See POSIX-SMB2 2.2.14.2.16 - * Link: https://gitlab.com/samba-team/smb3-posix-spec/-/blob/master/smb3_posix_extensions.md + * See POSIX-SMB2 2.1.3.2.1 */ struct create_posix_rsp { u32 nlink; @@ -238,6 +237,7 @@ struct create_posix_rsp { #define SMB2_QUERY_DIRECTORY_IOV_SIZE 2 /* + * See POSIX-FSCC 2.2.1 * SMB2-only POSIX info level for query dir * * See posix_info_sid_size(), posix_info_extra_size() and @@ -256,13 +256,17 @@ struct smb2_posix_info { __le64 Inode; __le32 DeviceId; __le32 Zero; - /* beginning of POSIX Create Context Response */ + /* + * Beginning of POSIX Create Context Response + * See POSIX-SMB2 2.1.3.2.1 + */ __le32 HardLinks; __le32 ReparseTag; __le32 Mode; /* * var sized owner SID * var sized group SID + * End of POSIX Create Context Response * le32 filenamelength * u8 filename[] */ diff --git a/fs/smb/client/transport.c b/fs/smb/client/transport.c index e266859818a4..6e21b5f8754a 100644 --- a/fs/smb/client/transport.c +++ b/fs/smb/client/transport.c @@ -805,6 +805,18 @@ cifs_cancelled_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid) release_mid(server, mid); } +static void +cifs_mark_compound_mids_cancelled(struct mid_q_entry **mid, int count) +{ + int i; + + for (i = 0; i < count; i++) { + spin_lock(&mid[i]->mid_lock); + mid[i]->wait_cancelled = true; + spin_unlock(&mid[i]->mid_lock); + } +} + /* * cifs_pick_channel - pick an eligible channel for network operations * @@ -865,6 +877,7 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, int *resp_buf_type, struct kvec *resp_iov) { int i, j, optype, rc = 0; + int num_processed = 0; struct mid_q_entry *mid[MAX_COMPOUND]; bool cancelled_mid[MAX_COMPOUND] = {false}; struct cifs_credits credits[MAX_COMPOUND] = { @@ -965,6 +978,10 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, if (rc < 0) { revert_current_mid(server, num_rqst); server->sequence_number -= 2; + for (i = 0; i < num_rqst; i++) { + delete_mid(server, mid[i]); + cancelled_mid[i] = true; + } } cifs_server_unlock(server); @@ -1011,6 +1028,14 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, break; } if (rc != 0) { + /* + * A completed CREATE earlier in the compound chain may have + * opened a remote handle even though a later wait was + * interrupted. Mark it cancelled so __release_mid() invokes + * the existing unmatched-open cleanup. + */ + cifs_mark_compound_mids_cancelled(mid, i); + for (; i < num_rqst; i++) { cifs_server_dbg(FYI, "Cancelling wait for mid %llu cmd: %d\n", mid[i]->mid, le16_to_cpu(mid[i]->command)); @@ -1033,6 +1058,14 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, rc = cifs_sync_mid_result(mid[i], server); if (rc != 0) { + /* + * A previous CREATE may have completed before this + * response failed. Mark it cancelled so its remote + * handle is closed when the mid is released. + */ + cifs_mark_compound_mids_cancelled(mid, i); + /* Keep their response buffers for cancelled-mid cleanup. */ + num_processed = 0; /* mark this mid as cancelled to not free it below */ cancelled_mid[i] = true; goto out; @@ -1042,13 +1075,24 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, mid[i]->mid_state != MID_RESPONSE_READY) { rc = smb_EIO1(smb_eio_trace_rx_mid_unready, mid[i]->mid_state); cifs_dbg(FYI, "Bad MID state?\n"); + cifs_mark_compound_mids_cancelled(mid, i); + num_processed = 0; goto out; } rc = server->ops->check_receive(mid[i], server, flags & CIFS_LOG_ERROR); + num_processed = i + 1; + } - if (resp_iov) { +out: + /* + * Delay moving response buffers out of their mids until response + * synchronization completes. This lets cancelled-mid cleanup inspect + * an earlier CREATE response if a later MID fails. + */ + if (resp_iov) { + for (i = 0; i < num_processed; i++) { buf = (char *)mid[i]->resp_buf; resp_iov[i].iov_base = buf; resp_iov[i].iov_len = mid[i]->resp_buf_size; @@ -1067,21 +1111,22 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, /* * Compounding is never used during session establish. */ - spin_lock(&ses->ses_lock); - if ((ses->ses_status == SES_NEW) || (optype & CIFS_NEG_OP) || (optype & CIFS_SESS_OP)) { - struct kvec iov = { - .iov_base = resp_iov[0].iov_base, - .iov_len = resp_iov[0].iov_len - }; - spin_unlock(&ses->ses_lock); - cifs_server_lock(server); - smb311_update_preauth_hash(ses, server, &iov, 1); - cifs_server_unlock(server); + if (num_processed == num_rqst) { spin_lock(&ses->ses_lock); + if ((ses->ses_status == SES_NEW) || (optype & CIFS_NEG_OP) || (optype & CIFS_SESS_OP)) { + struct kvec iov = { + .iov_base = resp_iov[0].iov_base, + .iov_len = resp_iov[0].iov_len + }; + spin_unlock(&ses->ses_lock); + cifs_server_lock(server); + smb311_update_preauth_hash(ses, server, &iov, 1); + cifs_server_unlock(server); + spin_lock(&ses->ses_lock); + } + spin_unlock(&ses->ses_lock); } - spin_unlock(&ses->ses_lock); -out: /* * This will dequeue all mids. After this it is important that the * demultiplex_thread will not process any of these mids any further. diff --git a/fs/squashfs/xz_wrapper.c b/fs/squashfs/xz_wrapper.c index 0a4ff3ec9c8c..6610af241449 100644 --- a/fs/squashfs/xz_wrapper.c +++ b/fs/squashfs/xz_wrapper.c @@ -57,10 +57,10 @@ static void *squashfs_xz_comp_opts(struct squashfs_sb_info *msblk, opts->dict_size = le32_to_cpu(comp_opts->dictionary_size); - /* the dictionary size should be 2^n or 2^n+2^(n+1) */ + /* the dictionary size should be positive and 2^n or 2^n+2^(n+1) */ n = ffs(opts->dict_size) - 1; - if (opts->dict_size != (1 << n) && opts->dict_size != (1 << n) + - (1 << (n + 1))) { + if (opts->dict_size <= 0 || (opts->dict_size != (1 << n) && + opts->dict_size != (1 << n) + (1 << (n + 1)))) { err = -EIO; goto out; } diff --git a/fs/super.c b/fs/super.c index 9d4025213521..1d5ccf540a9b 100644 --- a/fs/super.c +++ b/fs/super.c @@ -419,15 +419,19 @@ static struct super_block *alloc_super(struct file_system_type *type, int flags, void put_super(struct super_block *s) { if (refcount_dec_and_test(&s->s_passive)) { + struct file_system_type *type = s->s_type; spin_lock(&sb_lock); list_del_init(&s->s_list); + hlist_del_init(&s->s_instances); spin_unlock(&sb_lock); WARN_ON(s->s_dentry_lru.node); WARN_ON(s->s_inode_lru.node); WARN_ON(s->s_mounts); call_rcu(&s->rcu, destroy_super_rcu); + /* The unlink above may touch type->fs_supers, so drop it last. */ + put_filesystem(type); } } @@ -544,17 +548,6 @@ static void kill_super_notify(struct super_block *sb) if (sb->s_flags & SB_DEAD) return; - /* - * Remove it from @fs_supers so it isn't found by new - * sget_fc() walkers anymore. Any concurrent mounter still - * managing to grab a temporary reference is guaranteed to - * already see SB_DYING and will wait until we notify them about - * SB_DEAD. - */ - spin_lock(&sb_lock); - hlist_del_init(&sb->s_instances); - spin_unlock(&sb_lock); - /* Drop sget_fc()'s claim; a never-registered entry stays with the sb. */ if (sb->s_super_dev->sd_dev) { super_dev_put(sb->s_super_dev); @@ -563,11 +556,15 @@ static void kill_super_notify(struct super_block *sb) /* * Let concurrent mounts know that this thing is really dead. - * We don't need @sb->s_umount here as every concurrent caller - * will see SB_DYING and either discard the superblock or wait - * for SB_DEAD. + * sget_fc() skips SB_DEAD superblocks and calls test() under + * sb_lock, so set it under sb_lock: once we return no test() + * runs on this superblock anymore and none will start. Everyone + * else already saw SB_DYING and either discarded the superblock + * or waits for SB_DEAD. */ + spin_lock(&sb_lock); super_wake(sb, SB_DEAD); + spin_unlock(&sb_lock); } /** @@ -594,7 +591,6 @@ void deactivate_locked_super(struct super_block *s) list_lru_destroy(&s->s_dentry_lru); list_lru_destroy(&s->s_inode_lru); - put_filesystem(fs); put_super(s); } else { super_unlock_excl(s); @@ -781,12 +777,12 @@ void generic_shutdown_super(struct super_block *sb) } /* * Broadcast to everyone that grabbed a temporary reference to this - * superblock before we removed it from @fs_supers that the superblock - * is dying. Every walker of @fs_supers outside of sget_fc() will now - * discard this superblock and treat it as dead. + * superblock that it is dying. Every walker of @fs_supers outside + * of sget_fc() will now discard this superblock and treat it as + * dead. * - * We leave the superblock on @fs_supers so it can be found by - * sget_fc() until we passed sb->kill_sb(). + * sget_fc() keeps finding the superblock until SB_DEAD is set, so + * a concurrent mounter waits until we passed sb->kill_sb(). */ super_wake(sb, SB_DYING); super_unlock_excl(sb); @@ -865,6 +861,9 @@ struct super_block *sget_fc(struct fs_context *fc, spin_lock(&sb_lock); if (test) { hlist_for_each_entry(old, &fc->fs_type->fs_supers, s_instances) { + /* Only unlinked at the last passive reference. */ + if (super_flags(old, SB_DEAD)) + continue; if (test(old, fc)) goto share_extant_sb; } diff --git a/fs/xfs/libxfs/xfs_ag.h b/fs/xfs/libxfs/xfs_ag.h index fd22fe598931..ee636b66a72f 100644 --- a/fs/xfs/libxfs/xfs_ag.h +++ b/fs/xfs/libxfs/xfs_ag.h @@ -207,7 +207,7 @@ xfs_perag_next( } /* - * Per-ag geometry infomation and validation + * Per-ag geometry information and validation */ xfs_agblock_t xfs_ag_block_count(struct xfs_mount *mp, xfs_agnumber_t agno); void xfs_agino_range(struct xfs_mount *mp, xfs_agnumber_t agno, diff --git a/fs/xfs/libxfs/xfs_alloc.c b/fs/xfs/libxfs/xfs_alloc.c index d99602bcc16f..f762dcce8d13 100644 --- a/fs/xfs/libxfs/xfs_alloc.c +++ b/fs/xfs/libxfs/xfs_alloc.c @@ -3487,7 +3487,7 @@ xfs_alloc_read_agf( } /* - * Pre-proces allocation arguments to set initial state that we don't require + * Pre-process allocation arguments to set initial state that we don't require * callers to set up correctly, as well as bounds check the allocation args * that are set up. */ @@ -3608,7 +3608,7 @@ xfs_alloc_vextent_finish( * ABBA AGF deadlocks because a future allocation attempt in this * transaction may attempt to lock a lower number AGF. * - * We can't release the AGF until the transaction is commited, so at + * We can't release the AGF until the transaction is committed, so at * this point we must update the "first allocation" tracker to point at * this AG if the tracker is empty or points to a lower AG. This allows * the next allocation attempt to be modified appropriately to avoid diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c index b6288395f853..2c80f4fd0b78 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.c +++ b/fs/xfs/libxfs/xfs_attr_leaf.c @@ -1715,7 +1715,7 @@ xfs_attr3_leaf_add_work( /* * This freemap entry starts at the old end of the * leaf entry array, so we need to adjust its base - * upward to accomodate the larger array. + * upward to accommodate the larger array. */ diff = sizeof(struct xfs_attr_leaf_entry); } else if (ichdr->freemap[i].size > 0 && diff --git a/fs/xfs/libxfs/xfs_errortag.h b/fs/xfs/libxfs/xfs_errortag.h index 6de207fed2d8..f0c83f1f0b3b 100644 --- a/fs/xfs/libxfs/xfs_errortag.h +++ b/fs/xfs/libxfs/xfs_errortag.h @@ -83,7 +83,7 @@ #define XFS_RANDOM_DEFAULT 100 /* - * Table of errror injection knobs. The parameters to the XFS_ERRTAG macro are: + * Table of error injection knobs. The parameters to the XFS_ERRTAG macro are: * 1. The XFS_ERRTAG_ flag but without the prefix; * 2. The name of the sysfs knob; and * 3. The default value for the knob. diff --git a/fs/xfs/libxfs/xfs_exchmaps.c b/fs/xfs/libxfs/xfs_exchmaps.c index 49eda8d0994d..6a66b6075e0a 100644 --- a/fs/xfs/libxfs/xfs_exchmaps.c +++ b/fs/xfs/libxfs/xfs_exchmaps.c @@ -395,7 +395,7 @@ xfs_exchmaps_one_step( /* * Re-add both mappings. We exchange the file offsets between the two * maps and add the opposite map, which has the effect of filling the - * logical offsets we just unmapped, but with with the physical mapping + * logical offsets we just unmapped, but with the physical mapping * information exchanged. */ swap(irec1->br_startoff, irec2->br_startoff); @@ -959,16 +959,6 @@ xmi_can_exchange_reflink_flags( { struct xfs_mount *mp = req->ip1->i_mount; - /* - * The INO1_WRITTEN optimization can skip exchanging hole and - * unwritten mappings, which means we cannot guarantee that all - * shared extents actually moved to the other file. Clearing the - * reflink flag of an inode that still holds shared extents breaks - * the CoW write path, so refuse to exchange the flags in that case. - */ - if (req->flags & XFS_EXCHMAPS_INO1_WRITTEN) - return false; - /* * The INO1_WRITTEN optimization can skip exchanging hole and * unwritten mappings, which means we cannot guarantee that all diff --git a/fs/xfs/libxfs/xfs_format.h b/fs/xfs/libxfs/xfs_format.h index dd0ed046fbe9..1a7a7e60a170 100644 --- a/fs/xfs/libxfs/xfs_format.h +++ b/fs/xfs/libxfs/xfs_format.h @@ -1051,7 +1051,7 @@ enum xfs_dinode_fmt { * block is 1KB in size. * * With XFS_MAX_EXTCNT_DATA_FORK_SMALL representing maximum extent count and - * with 1KB sized blocks, a file can reach upto, + * with 1KB sized blocks, a file can reach up to, * 1KB * (2^31) = 2TB * * This is much larger than the theoretical maximum size of a directory diff --git a/fs/xfs/libxfs/xfs_inode_buf.c b/fs/xfs/libxfs/xfs_inode_buf.c index e4c3f7b24e95..0340e2189921 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.c +++ b/fs/xfs/libxfs/xfs_inode_buf.c @@ -626,7 +626,7 @@ xfs_dinode_verify( * have di_nlink track the link count, even if the actual filesystem * only supported V1 inodes (i.e. di_onlink). When writing out the * ondisk inode, it would set both the ondisk di_nlink and di_onlink to - * the the incore di_nlink value, which is why we cannot check for + * the incore di_nlink value, which is why we cannot check for * di_nlink==0 on a V1 inode. V2/3 inodes would get written out with * di_onlink==0, so we can check that. */ diff --git a/fs/xfs/libxfs/xfs_metafile.c b/fs/xfs/libxfs/xfs_metafile.c index 71f004e9dc64..1f54d39003c2 100644 --- a/fs/xfs/libxfs/xfs_metafile.c +++ b/fs/xfs/libxfs/xfs_metafile.c @@ -297,14 +297,14 @@ xfs_metafile_resv_init( goto out_unlock; /* - * Space taken by the per-AG metadata btrees are accounted on-disk as - * used space. We therefore only hide the space that is reserved but - * not used by the trees. + * Space taken by metadata btrees are accounted on-disk as used space. + * We therefore only hide the space that is reserved but not used by + * the trees. */ if (used > target) target = used; else if (target > dblocks_avail) - target = dblocks_avail; + target = max(dblocks_avail, used); hidden_space = target - used; error = xfs_dec_fdblocks(mp, hidden_space, true); diff --git a/fs/xfs/libxfs/xfs_rtrefcount_btree.c b/fs/xfs/libxfs/xfs_rtrefcount_btree.c index e2950dbe2068..dcc89b8e149b 100644 --- a/fs/xfs/libxfs/xfs_rtrefcount_btree.c +++ b/fs/xfs/libxfs/xfs_rtrefcount_btree.c @@ -617,7 +617,7 @@ xfs_rtrefcountbt_from_disk( fpp = xfs_rtrefcount_droot_ptr_addr(dblock, 1, maxrecs); tpp = xfs_rtrefcount_broot_ptr_addr(mp, rblock, 1, rblocklen); numrecs = be16_to_cpu(dblock->bb_numrecs); - memcpy(tkp, fkp, 2 * sizeof(*fkp) * numrecs); + memcpy(tkp, fkp, sizeof(*fkp) * numrecs); memcpy(tpp, fpp, sizeof(*fpp) * numrecs); } else { frp = xfs_rtrefcount_droot_rec_addr(dblock, 1); @@ -703,7 +703,7 @@ xfs_rtrefcountbt_to_disk( fpp = xfs_rtrefcount_broot_ptr_addr(mp, rblock, 1, rblocklen); tpp = xfs_rtrefcount_droot_ptr_addr(dblock, 1, maxrecs); numrecs = be16_to_cpu(rblock->bb_numrecs); - memcpy(tkp, fkp, 2 * sizeof(*fkp) * numrecs); + memcpy(tkp, fkp, sizeof(*fkp) * numrecs); memcpy(tpp, fpp, sizeof(*fpp) * numrecs); } else { frp = xfs_rtrefcount_rec_addr(rblock, 1); diff --git a/fs/xfs/scrub/agheader_repair.c b/fs/xfs/scrub/agheader_repair.c index a66b611588c4..ff1b4b361cf2 100644 --- a/fs/xfs/scrub/agheader_repair.c +++ b/fs/xfs/scrub/agheader_repair.c @@ -1369,7 +1369,7 @@ xrep_iunlink_mark_ondisk( /* * Walk an iunlink bucket's inode list. For each inode that should be on this - * chain, clear its entry in in iunlink_bmp because it's ok and we don't need + * chain, clear its entry in iunlink_bmp because it's ok and we don't need * to touch it further. */ STATIC int diff --git a/fs/xfs/scrub/alloc_repair.c b/fs/xfs/scrub/alloc_repair.c index 95e318e4f3a6..2398e3819597 100644 --- a/fs/xfs/scrub/alloc_repair.c +++ b/fs/xfs/scrub/alloc_repair.c @@ -338,7 +338,7 @@ xrep_cntbt_extent_cmp( } /* - * Sort the free extents by length so so that we can put the records into the + * Sort the free extents by length so that we can put the records into the * cntbt in the correct order. Don't let userspace kill us if we're resorting * after allocating btree blocks. */ diff --git a/fs/xfs/scrub/bitmap.c b/fs/xfs/scrub/bitmap.c index c7fa908d92b2..08f216d26ec6 100644 --- a/fs/xfs/scrub/bitmap.c +++ b/fs/xfs/scrub/bitmap.c @@ -122,8 +122,8 @@ xbitmap64_set( uint64_t start, uint64_t len) { - struct xbitmap64_node *left; - struct xbitmap64_node *right; + struct xbitmap64_node *left = NULL; + struct xbitmap64_node *right = NULL; uint64_t last = start + len - 1; int error; @@ -131,6 +131,7 @@ xbitmap64_set( left = xbitmap64_tree_iter_first(&bitmap->xb_root, start, last); if (left && left->bn_start <= start && left->bn_last >= last) return 0; + left = NULL; /* Clear out everything in the range we want to set. */ error = xbitmap64_clear(bitmap, start, len); @@ -138,11 +139,15 @@ xbitmap64_set( return error; /* Do we have a left-adjacent extent? */ - left = xbitmap64_tree_iter_first(&bitmap->xb_root, start - 1, start - 1); + if (start > 0) + left = xbitmap64_tree_iter_first(&bitmap->xb_root, start - 1, + start - 1); ASSERT(!left || left->bn_last + 1 == start); /* Do we have a right-adjacent extent? */ - right = xbitmap64_tree_iter_first(&bitmap->xb_root, last + 1, last + 1); + if (last < U64_MAX) + right = xbitmap64_tree_iter_first(&bitmap->xb_root, last + 1, + last + 1); ASSERT(!right || right->bn_start == last + 1); if (left && right) { @@ -397,8 +402,8 @@ xbitmap32_set( uint32_t start, uint32_t len) { - struct xbitmap32_node *left; - struct xbitmap32_node *right; + struct xbitmap32_node *left = NULL; + struct xbitmap32_node *right = NULL; uint32_t last = start + len - 1; int error; @@ -406,6 +411,7 @@ xbitmap32_set( left = xbitmap32_tree_iter_first(&bitmap->xb_root, start, last); if (left && left->bn_start <= start && left->bn_last >= last) return 0; + left = NULL; /* Clear out everything in the range we want to set. */ error = xbitmap32_clear(bitmap, start, len); @@ -413,11 +419,15 @@ xbitmap32_set( return error; /* Do we have a left-adjacent extent? */ - left = xbitmap32_tree_iter_first(&bitmap->xb_root, start - 1, start - 1); + if (start > 0) + left = xbitmap32_tree_iter_first(&bitmap->xb_root, start - 1, + start - 1); ASSERT(!left || left->bn_last + 1 == start); /* Do we have a right-adjacent extent? */ - right = xbitmap32_tree_iter_first(&bitmap->xb_root, last + 1, last + 1); + if (last < U32_MAX) + right = xbitmap32_tree_iter_first(&bitmap->xb_root, last + 1, + last + 1); ASSERT(!right || right->bn_start == last + 1); if (left && right) { diff --git a/fs/xfs/scrub/dir.c b/fs/xfs/scrub/dir.c index 2a037aae904d..19d974c7e2b7 100644 --- a/fs/xfs/scrub/dir.c +++ b/fs/xfs/scrub/dir.c @@ -492,7 +492,7 @@ xchk_directory_data_bestfree( goto out; xchk_buffer_recheck(sc, bp); - if (xfs_has_crc(sc->mp)) { + if (!is_block && xfs_has_crc(sc->mp)) { struct xfs_dir3_data_hdr *hdr3 = bp->b_addr; if (hdr3->pad) diff --git a/fs/xfs/scrub/dirtree.c b/fs/xfs/scrub/dirtree.c index 9b0ab2316612..887383d2e941 100644 --- a/fs/xfs/scrub/dirtree.c +++ b/fs/xfs/scrub/dirtree.c @@ -1021,7 +1021,7 @@ xchk_dirtree( return error; } -/* Does the directory targetted by this scrub have no parents? */ +/* Does the directory targeted by this scrub have no parents? */ bool xchk_dirtree_parentless(const struct xchk_dirtree *dl) { diff --git a/fs/xfs/scrub/findparent.c b/fs/xfs/scrub/findparent.c index eab3ac2704be..d921fe5a9b0c 100644 --- a/fs/xfs/scrub/findparent.c +++ b/fs/xfs/scrub/findparent.c @@ -473,6 +473,9 @@ xrep_findparent_from_dcache( pip = igrab(d_inode(parent)); dput(parent); + if (!pip) + goto out_dput; + if (S_ISDIR(pip->i_mode)) { ret = pip->i_ino; trace_xrep_findparent_from_dcache(sc->ip, ret); diff --git a/fs/xfs/scrub/health.c b/fs/xfs/scrub/health.c index 2171bcf0f6c1..487ecc5f9f3c 100644 --- a/fs/xfs/scrub/health.c +++ b/fs/xfs/scrub/health.c @@ -202,9 +202,9 @@ xchk_update_health( * there's no sick flag defined for it, so we branch here ahead of the * mask check. */ - if (sc->sm->sm_type == XFS_SCRUB_TYPE_HEALTHY && - !(sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) { - xchk_mark_all_healthy(sc->mp); + if (sc->sm->sm_type == XFS_SCRUB_TYPE_HEALTHY) { + if (!(sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) + xchk_mark_all_healthy(sc->mp); return; } diff --git a/fs/xfs/scrub/inode.c b/fs/xfs/scrub/inode.c index 65b13e311916..46e9bf4a4317 100644 --- a/fs/xfs/scrub/inode.c +++ b/fs/xfs/scrub/inode.c @@ -607,7 +607,7 @@ xchk_dinode( } /* di_forkoff */ - if (XFS_DFORK_BOFF(dip) >= mp->m_sb.sb_inodesize) + if (dip->di_forkoff >= (XFS_LITINO(mp) >> 3)) xchk_ino_set_corrupt(sc, ino); if (naextents != 0 && dip->di_forkoff == 0) xchk_ino_set_corrupt(sc, ino); diff --git a/fs/xfs/scrub/inode_repair.c b/fs/xfs/scrub/inode_repair.c index 8bc508336aa5..b87c22146233 100644 --- a/fs/xfs/scrub/inode_repair.c +++ b/fs/xfs/scrub/inode_repair.c @@ -1702,7 +1702,7 @@ xrep_inode_blockcounts( &acount); if (error) return error; - if (count >= sc->mp->m_sb.sb_dblocks) + if (acount >= sc->mp->m_sb.sb_dblocks) return -EFSCORRUPTED; error = xrep_ino_ensure_extent_count(sc, XFS_ATTR_FORK, nextents); diff --git a/fs/xfs/scrub/newbt.c b/fs/xfs/scrub/newbt.c index c82f4631fd9c..584076b2a6ee 100644 --- a/fs/xfs/scrub/newbt.c +++ b/fs/xfs/scrub/newbt.c @@ -193,9 +193,11 @@ xrep_newbt_add_blocks( struct xrep_newbt_resv *resv; int error; - resv = kmalloc_obj(struct xrep_newbt_resv, XCHK_GFP_FLAGS); - if (!resv) - return -ENOMEM; + /* + * We have no way to clean up the allocated space *and* return an + * ENOMEM if we fail to allocate this control structure. + */ + resv = kmalloc_obj(struct xrep_newbt_resv, GFP_KERNEL | __GFP_NOFAIL); INIT_LIST_HEAD(&resv->list); resv->agbno = XFS_FSB_TO_AGBNO(mp, args->fsbno); diff --git a/fs/xfs/scrub/orphanage.c b/fs/xfs/scrub/orphanage.c index 3aca66869b80..21e31eeaa042 100644 --- a/fs/xfs/scrub/orphanage.c +++ b/fs/xfs/scrub/orphanage.c @@ -192,12 +192,16 @@ xrep_orphanage_create( /* Make sure the orphanage is owned by root. */ error = xrep_chown_orphanage(sc, XFS_I(orphanage_inode)); if (error) - goto out_dput_orphanage; + goto out_rele_orphanage; /* Stash the reference for later and bail out. */ sc->orphanage = XFS_I(orphanage_inode); sc->orphanage_ilock_flags = 0; + orphanage_inode = NULL; +out_rele_orphanage: + if (orphanage_inode) + xchk_irele(sc, XFS_I(orphanage_inode)); out_dput_orphanage: end_creating(orphanage_dentry); out_dput_root: diff --git a/fs/xfs/scrub/reap.c b/fs/xfs/scrub/reap.c index f698b9be3dd1..0dfe61bafc6a 100644 --- a/fs/xfs/scrub/reap.c +++ b/fs/xfs/scrub/reap.c @@ -172,7 +172,7 @@ static inline bool xreap_is_dirty(const struct xreap_state *rs) } /* - * Decide if we need to roll the transaction to clear out the the log + * Decide if we need to roll the transaction to clear out the log * reservation that we allocated to buffer invalidations. */ static inline bool xreap_want_binval_roll(const struct xreap_state *rs) diff --git a/fs/xfs/scrub/repair.c b/fs/xfs/scrub/repair.c index 11697a8b2a1d..c2a437416227 100644 --- a/fs/xfs/scrub/repair.c +++ b/fs/xfs/scrub/repair.c @@ -399,6 +399,7 @@ xrep_calc_rtgroup_resblks( struct xfs_mount *mp = sc->mp; struct xfs_scrub_metadata *sm = sc->sm; uint64_t usedlen; + xfs_extlen_t refcbt_sz = 0; xfs_extlen_t rmapbt_sz = 0; if (!(sm->sm_flags & XFS_SCRUB_IFLAG_REPAIR)) @@ -411,13 +412,27 @@ xrep_calc_rtgroup_resblks( usedlen = xfs_rtbxlen_to_blen(mp, xfs_rtgroup_extents(mp, sm->sm_agno)); ASSERT(usedlen <= XFS_MAX_RGBLOCKS); + if (xfs_has_reflink(mp)) + refcbt_sz = xfs_rtrefcountbt_calc_size(mp, usedlen); + if (xfs_has_rmapbt(mp)) rmapbt_sz = xfs_rtrmapbt_calc_size(mp, usedlen); - trace_xrep_calc_rtgroup_resblks_btsize(mp, sm->sm_agno, usedlen, - rmapbt_sz); + /* + * Guess how many blocks we need to rebuild the rmapbt. For + * non-reflink filesystems we can't have more records than used blocks. + * However, with reflink it's possible to have more than one rmap + * record per rtgroup block. We don't know how many rmaps there could + * be in the rtgroup, so we start off with what we hope is an generous + * over-estimation. + */ + if (refcbt_sz > 0 && rmapbt_sz > 0) + rmapbt_sz *= 2; - return rmapbt_sz; + trace_xrep_calc_rtgroup_resblks_btsize(mp, sm->sm_agno, usedlen, + rmapbt_sz, refcbt_sz); + + return max(rmapbt_sz, refcbt_sz); } #endif /* CONFIG_XFS_RT */ diff --git a/fs/xfs/scrub/rmap.c b/fs/xfs/scrub/rmap.c index 0cd3eecd2ca5..68e2847c962b 100644 --- a/fs/xfs/scrub/rmap.c +++ b/fs/xfs/scrub/rmap.c @@ -493,11 +493,18 @@ xchk_rmapbt_walk_ag_metadata( * If there's an error, set XFAIL and disable the bitmap * cross-referencing checks, but proceed with the scrub anyway. */ - if (error) - xchk_btree_xref_process_error(sc, sc->sa.rmap_cur, - sc->sa.rmap_cur->bc_nlevels - 1, &error); - else - cr->bitmaps_complete = true; + if (error) { + if (!xchk_btree_xref_process_error(sc, sc->sa.rmap_cur, + sc->sa.rmap_cur->bc_nlevels - 1, &error)) { + /* only set incomplete if we didn't set xfail */ + if (error) + xchk_set_incomplete(sc); + } + + return 0; + } + + cr->bitmaps_complete = true; return 0; } @@ -567,7 +574,8 @@ xchk_rmapbt( if (error) goto out; - xchk_rmapbt_check_bitmaps(sc, cr); + if (cr->bitmaps_complete) + xchk_rmapbt_check_bitmaps(sc, cr); out: xagb_bitmap_destroy(&cr->refcbt_owned); diff --git a/fs/xfs/scrub/rmap_repair.c b/fs/xfs/scrub/rmap_repair.c index 590f9f41856e..725035bf4903 100644 --- a/fs/xfs/scrub/rmap_repair.c +++ b/fs/xfs/scrub/rmap_repair.c @@ -1109,6 +1109,7 @@ xrep_rmap_try_reserve( return error; error = xfs_agfl_walk(sc->mp, agf, agfl_bp, xrep_rmap_walk_agfl, &ra); + xfs_trans_brelse(sc->tp, agfl_bp); if (error) return error; diff --git a/fs/xfs/scrub/scrub.h b/fs/xfs/scrub/scrub.h index 737a5d6db15f..b093945f3631 100644 --- a/fs/xfs/scrub/scrub.h +++ b/fs/xfs/scrub/scrub.h @@ -40,7 +40,7 @@ static inline int xchk_maybe_relax(struct xchk_relax *widget) return 0; widget->resched_nr = 0; - if (unlikely(widget->next_resched <= jiffies)) { + if (unlikely(time_after_eq(jiffies, widget->next_resched))) { cond_resched(); widget->next_resched = XCHK_RELAX_NEXT; } diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 0f5adc293962..cb85f75ce101 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -2376,25 +2376,29 @@ TRACE_EVENT(xrep_calc_ag_resblks_btsize, #ifdef CONFIG_XFS_RT TRACE_EVENT(xrep_calc_rtgroup_resblks_btsize, TP_PROTO(struct xfs_mount *mp, xfs_rgnumber_t rgno, - xfs_rgblock_t usedlen, xfs_rgblock_t rmapbt_sz), - TP_ARGS(mp, rgno, usedlen, rmapbt_sz), + xfs_rgblock_t usedlen, xfs_rgblock_t rmapbt_sz, + xfs_rgblock_t refcbt_sz), + TP_ARGS(mp, rgno, usedlen, rmapbt_sz, refcbt_sz), TP_STRUCT__entry( __field(dev_t, dev) __field(xfs_rgnumber_t, rgno) __field(xfs_rgblock_t, usedlen) __field(xfs_rgblock_t, rmapbt_sz) + __field(xfs_rgblock_t, refcbt_sz) ), TP_fast_assign( __entry->dev = mp->m_super->s_dev; __entry->rgno = rgno; __entry->usedlen = usedlen; __entry->rmapbt_sz = rmapbt_sz; + __entry->refcbt_sz = refcbt_sz; ), - TP_printk("dev %d:%d rgno 0x%x usedlen %u rmapbt %u", + TP_printk("dev %d:%d rgno 0x%x usedlen %u rmapbt %u refcountbt %u", MAJOR(__entry->dev), MINOR(__entry->dev), __entry->rgno, __entry->usedlen, - __entry->rmapbt_sz) + __entry->rmapbt_sz, + __entry->refcbt_sz) ); #endif /* CONFIG_XFS_RT */ diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c index 89f6e79a955f..aa5b41629747 100644 --- a/fs/xfs/xfs_bmap_item.c +++ b/fs/xfs/xfs_bmap_item.c @@ -339,7 +339,7 @@ xfs_bmap_update_get_group( /* * Bump the intent count on behalf of the deferred rmap and refcount - * intent items that that we can queue when we finish this bmap work. + * intent items that we can queue when we finish this bmap work. * This new intent item will bump the intent count before the bmap * intent drops the intent count, ensuring that the intent count * remains nonzero across the transaction roll. diff --git a/fs/xfs/xfs_dquot.c b/fs/xfs/xfs_dquot.c index b4f6c594808c..e696ee36c2e8 100644 --- a/fs/xfs/xfs_dquot.c +++ b/fs/xfs/xfs_dquot.c @@ -139,10 +139,14 @@ xfs_qm_adjust_dqlimits( dq->q_ino.softlimit = defq->ino.soft; if (!dq->q_ino.hardlimit) dq->q_ino.hardlimit = defq->ino.hard; - if (!dq->q_rtb.softlimit) + if (!dq->q_rtb.softlimit) { dq->q_rtb.softlimit = defq->rtb.soft; - if (!dq->q_rtb.hardlimit) + prealloc = 1; + } + if (!dq->q_rtb.hardlimit) { dq->q_rtb.hardlimit = defq->rtb.hard; + prealloc = 1; + } if (prealloc) xfs_dquot_set_prealloc_limits(dq); diff --git a/fs/xfs/xfs_exchrange.c b/fs/xfs/xfs_exchrange.c index c69ecd6a19de..fafb4e3f065c 100644 --- a/fs/xfs/xfs_exchrange.c +++ b/fs/xfs/xfs_exchrange.c @@ -633,6 +633,9 @@ xfs_exchrange_prep( if (error) return error; + if (fxr->flags & XFS_EXCHANGE_RANGE_DRY_RUN) + return 0; + trace_xfs_exchrange_flush(fxr, ip1, ip2); /* Flush the relevant ranges of both files. */ @@ -709,9 +712,11 @@ xfs_exchrange_contents( * other file write would do. This may involve turning on support for * logged xattrs if either file has security capabilities. */ - error = xfs_exchange_range_finish(fxr); - if (error) - goto out_unlock; + if (!(fxr->flags & XFS_EXCHANGE_RANGE_DRY_RUN)) { + error = xfs_exchange_range_finish(fxr); + if (error) + goto out_unlock; + } out_unlock: xfs_iunlock2_io_mmap(ip1, ip2); @@ -902,7 +907,7 @@ xfs_ioc_commit_range( if (copy_from_user(&args, argp, sizeof(args))) return -EFAULT; - if (args.flags & ~XFS_EXCHANGE_RANGE_ALL_FLAGS) + if (args.pad || (args.flags & ~XFS_EXCHANGE_RANGE_ALL_FLAGS)) return -EINVAL; if (kern_f->magic != XCR_FRESH_MAGIC) return -EBUSY; diff --git a/fs/xfs/xfs_healthmon.c b/fs/xfs/xfs_healthmon.c index c3749675ef19..2bdd747f1ead 100644 --- a/fs/xfs/xfs_healthmon.c +++ b/fs/xfs/xfs_healthmon.c @@ -247,7 +247,9 @@ xfs_healthmon_merge_events( case XFS_HEALTHMON_DIOWRITE: case XFS_HEALTHMON_DATALOST: /* logically adjacent file ranges can merge */ - if (existing->fino != new->fino || existing->fgen != new->fgen) + if (existing->fino != new->fino || + existing->fgen != new->fgen || + existing->error != new->error) return false; if (existing->fpos + existing->flen == new->fpos) { diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 82dac88e3c4c..de8be344e987 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -1653,7 +1653,7 @@ xfs_blockgc_free_dquots( do_work = true; } - if (XFS_IS_UQUOTA_ENFORCED(mp) && gdqp && xfs_dquot_lowsp(gdqp)) { + if (XFS_IS_GQUOTA_ENFORCED(mp) && gdqp && xfs_dquot_lowsp(gdqp)) { icw.icw_gid = make_kgid(mp->m_super->s_user_ns, gdqp->q_id); icw.icw_flags |= XFS_ICWALK_FLAG_GID; do_work = true; diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 030a7c8f2c12..621513d7215e 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -2669,7 +2669,7 @@ xfs_irele( } /* - * Ensure all commited transactions touching the inode are written to the log. + * Ensure all committed transactions touching the inode are written to the log. */ int xfs_log_force_inode( diff --git a/fs/xfs/xfs_log_cil.c b/fs/xfs/xfs_log_cil.c index f9e07a32f60f..9446ac44ba88 100644 --- a/fs/xfs/xfs_log_cil.c +++ b/fs/xfs/xfs_log_cil.c @@ -1370,7 +1370,7 @@ xlog_cil_cleanup_whiteouts( * allocation context. However, we do not want to block on memory reclaim * recursing back into the filesystem because this push may have been triggered * by memory reclaim itself. Hence we really need to run under full GFP_NOFS - * contraints here. + * constraints here. */ static void xlog_cil_push_work( diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index e7e49529658b..cf0d610265fe 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -2736,12 +2736,13 @@ xlog_recover_iunlink_bucket( { struct xfs_mount *mp = pag_mount(pag); struct xfs_inode *prev_ip = NULL; - struct xfs_inode *ip; xfs_agino_t prev_agino, agino; int error = 0; agino = be32_to_cpu(agi->agi_unlinked[bucket]); while (agino != NULLAGINO) { + struct xfs_inode *ip; + error = xfs_iget(mp, NULL, xfs_agino_to_ino(pag, agino), 0, 0, &ip); if (error) @@ -2750,11 +2751,11 @@ xlog_recover_iunlink_bucket( ASSERT(VFS_I(ip)->i_nlink == 0); ASSERT(VFS_I(ip)->i_mode != 0); xfs_iflags_clear(ip, XFS_IRECOVERY); - agino = ip->i_next_unlinked; if (prev_ip) { ip->i_prev_unlinked = prev_agino; xfs_irele(prev_ip); + prev_ip = NULL; /* * Ensure the inode is removed from the unlinked list @@ -2766,18 +2767,20 @@ xlog_recover_iunlink_bucket( * complete. */ error = xfs_inodegc_flush(mp); - if (error) - break; + if (error) { + xfs_irele(ip); + return error; + } } prev_agino = agino; + agino = ip->i_next_unlinked; prev_ip = ip; } if (prev_ip) { int error2; - ip->i_prev_unlinked = prev_agino; xfs_irele(prev_ip); error2 = xfs_inodegc_flush(mp); diff --git a/fs/xfs/xfs_platform.h b/fs/xfs/xfs_platform.h index 5d542e95fe44..745d715b4c64 100644 --- a/fs/xfs/xfs_platform.h +++ b/fs/xfs/xfs_platform.h @@ -153,7 +153,7 @@ static inline void delay(long ticks) /* * XFS wrapper structure for sysfs support. It depends on external data * structures and is embedded in various internal data structures to implement - * the XFS sysfs object heirarchy. Define it here for broad access throughout + * the XFS sysfs object hierarchy. Define it here for broad access throughout * the codebase. */ struct xfs_kobj { diff --git a/fs/xfs/xfs_qm.c b/fs/xfs/xfs_qm.c index 99a82107b8e6..54d00d543b51 100644 --- a/fs/xfs/xfs_qm.c +++ b/fs/xfs/xfs_qm.c @@ -1432,16 +1432,22 @@ xfs_qm_flush_one( error = xfs_dquot_use_attached_buf(dqp, &bp); if (error) - goto out_unlock; + goto out_dqflock; if (!bp) { error = -EFSCORRUPTED; - goto out_unlock; + goto out_dqflock; } error = xfs_qm_dqflush(dqp, bp); if (!error) xfs_buf_delwri_queue(bp, buffer_list); xfs_buf_relse(bp); + mutex_unlock(&dqp->q_qlock); + xfs_qm_dqrele(dqp); + return error; + +out_dqflock: + xfs_dqfunlock(dqp); out_unlock: mutex_unlock(&dqp->q_qlock); xfs_qm_dqrele(dqp); diff --git a/fs/xfs/xfs_refcount_item.c b/fs/xfs/xfs_refcount_item.c index 8bccf89a7766..682c6e1b45e3 100644 --- a/fs/xfs/xfs_refcount_item.c +++ b/fs/xfs/xfs_refcount_item.c @@ -508,6 +508,7 @@ xfs_refcount_recover_work( struct xfs_cui_log_item *cuip = CUI_ITEM(lip); struct xfs_trans *tp; struct xfs_mount *mp = lip->li_log->l_mp; + unsigned int dblocks; bool isrt = xfs_cui_item_isrt(lip); int i; int error = 0; @@ -543,8 +544,11 @@ xfs_refcount_recover_work( * full btree split on either end of the refcount range. */ resv = xlog_recover_resv(&M_RES(mp)->tr_itruncate); - error = xfs_trans_alloc(mp, &resv, mp->m_refc_maxlevels * 2, 0, - XFS_TRANS_RESERVE, &tp); + if (isrt) + dblocks = mp->m_rtrefc_maxlevels * 2; + else + dblocks = mp->m_refc_maxlevels * 2; + error = xfs_trans_alloc(mp, &resv, dblocks, 0, XFS_TRANS_RESERVE, &tp); if (error) return error; diff --git a/fs/xfs/xfs_reflink.h b/fs/xfs/xfs_reflink.h index 9d1ed9bb0bee..683c1841e640 100644 --- a/fs/xfs/xfs_reflink.h +++ b/fs/xfs/xfs_reflink.h @@ -48,9 +48,6 @@ extern int xfs_reflink_end_cow(struct xfs_inode *ip, xfs_off_t offset, int xfs_reflink_end_atomic_cow(struct xfs_inode *ip, xfs_off_t offset, xfs_off_t count); extern int xfs_reflink_recover_cow(struct xfs_mount *mp); -extern loff_t xfs_reflink_remap_range(struct file *file_in, loff_t pos_in, - struct file *file_out, loff_t pos_out, loff_t len, - unsigned int remap_flags); extern int xfs_reflink_inode_has_shared_extents(struct xfs_trans *tp, struct xfs_inode *ip, bool *has_shared); extern int xfs_reflink_clear_inode_flag(struct xfs_inode *ip, diff --git a/fs/xfs/xfs_rmap_item.c b/fs/xfs/xfs_rmap_item.c index 2a3a73a8566d..000cff1ce324 100644 --- a/fs/xfs/xfs_rmap_item.c +++ b/fs/xfs/xfs_rmap_item.c @@ -573,6 +573,7 @@ xfs_rmap_recover_work( struct xfs_rui_log_item *ruip = RUI_ITEM(lip); struct xfs_trans *tp; struct xfs_mount *mp = lip->li_log->l_mp; + unsigned int dblocks; bool isrt = xfs_rui_item_isrt(lip); int i; int error = 0; @@ -596,8 +597,11 @@ xfs_rmap_recover_work( } resv = xlog_recover_resv(&M_RES(mp)->tr_itruncate); - error = xfs_trans_alloc(mp, &resv, mp->m_rmap_maxlevels, 0, - XFS_TRANS_RESERVE, &tp); + if (isrt) + dblocks = mp->m_rtrmap_maxlevels; + else + dblocks = mp->m_rmap_maxlevels; + error = xfs_trans_alloc(mp, &resv, dblocks, 0, XFS_TRANS_RESERVE, &tp); if (error) return error; diff --git a/fs/xfs/xfs_zone_alloc.c b/fs/xfs/xfs_zone_alloc.c index 28c1e48909fa..b75cf3bfe33c 100644 --- a/fs/xfs/xfs_zone_alloc.c +++ b/fs/xfs/xfs_zone_alloc.c @@ -820,7 +820,7 @@ xfs_get_cached_zone( spin_unlock(&ip->i_flags_lock); } - if (!atomic_inc_not_zero(&oz->oz_ref)) + if (oz && !atomic_inc_not_zero(&oz->oz_ref)) oz = NULL; out_unlock: rcu_read_unlock(); @@ -828,7 +828,7 @@ xfs_get_cached_zone( } /* - * Stash our zone in the inode so that is is reused for future allocations. + * Stash our zone in the inode so that it is reused for future allocations. * * The open_zone structure will be pinned until either the inode is freed or * until the cached open zone is replaced with a different one because the diff --git a/fs/xfs/xfs_zone_gc.c b/fs/xfs/xfs_zone_gc.c index 5fdcf98a2133..54b70ed2922f 100644 --- a/fs/xfs/xfs_zone_gc.c +++ b/fs/xfs/xfs_zone_gc.c @@ -46,7 +46,7 @@ * before remapping. * * Once a zone does not contain any valid data, be that through GC or user - * block removal, it is queued for for a zone reset. The reset operation + * block removal, it is queued for a zone reset. The reset operation * carefully ensures that the RT device cache is flushed and all transactions * referencing the rmap have been committed to disk. */ diff --git a/include/linux/bpf.h b/include/linux/bpf.h index e57af902560c..0ecb9418dfbd 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1258,11 +1258,15 @@ struct btf_func_model { #define BPF_TRAMP_F_INDIRECT BIT(8) /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 - * bytes on x86. + * bytes on x86. The trampoline image has to fit in PAGE_SIZE. */ enum { -#if defined(__s390x__) +#if defined(__s390x__) || defined(__powerpc64__) BPF_MAX_TRAMP_LINKS = 27, +#elif defined(__x86_64__) + BPF_MAX_TRAMP_LINKS = 36, +#elif defined(__aarch64__) + BPF_MAX_TRAMP_LINKS = 37, #else BPF_MAX_TRAMP_LINKS = 38, #endif @@ -1315,6 +1319,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i void *arch_alloc_bpf_trampoline(unsigned int size); void arch_free_bpf_trampoline(void *image, unsigned int size); int __must_check arch_protect_bpf_trampoline(void *image, unsigned int size); +int arch_bpf_trampoline_skip(void *nop, void *target); int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, struct bpf_tramp_nodes *tnodes, void *func_addr); @@ -1363,19 +1368,48 @@ enum bpf_tramp_prog_type { BPF_TRAMP_FSESSION, }; +/* + * Each prog call in a trampoline image is preceded by a nop. When the prog is + * detached, the nop is patched to a jump to target, right after the call, so + * that tasks still running in the image skip the prog. + */ +struct bpf_tramp_skip { + struct bpf_prog *prog; + void *nop; + void *target; +}; + struct bpf_tramp_image { void *image; int size; struct bpf_ksym ksym; struct percpu_ref pcref; - void *ip_after_call; - void *ip_epilogue; + bool call_orig; + /* entry in tr->images, the image holds a reference on tr */ + struct bpf_trampoline *tr; + struct list_head list; + int nr_skips; + struct bpf_tramp_skip *skips; union { struct rcu_head rcu; struct work_struct work; }; }; +static inline void bpf_tramp_image_add_skip(struct bpf_tramp_image *im, struct bpf_prog *prog, + void *nop, void *target) +{ + struct bpf_tramp_skip *skip; + + /* struct_ops trampolines and arch_bpf_trampoline_size() have no image */ + if (!im || !im->skips) + return; + skip = &im->skips[im->nr_skips++]; + skip->prog = prog; + skip->nop = nop; + skip->target = target; +} + struct bpf_trampoline { /* hlist for trampoline_key_table */ struct hlist_node hlist_key; @@ -1402,6 +1436,8 @@ struct bpf_trampoline { int progs_cnt[BPF_TRAMP_MAX]; /* Executable image of trampoline */ struct bpf_tramp_image *cur_image; + /* Images not freed yet, cur_image and older ones still in use */ + struct list_head images; /* Used as temporary old image storage for multi_attach */ struct { struct bpf_tramp_image *old_image; @@ -1651,8 +1687,9 @@ static inline void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags struct bpf_func_info_aux { u16 linkage; bool unreliable; - bool called : 1; - bool verified : 1; + /* Indexed by in_sleepable. */ + bool called[2]; + bool verified[2]; }; enum bpf_jit_poke_reason { @@ -1769,6 +1806,7 @@ struct bpf_prog_aux { bool offload_requested; /* Program is bound and offloaded to the netdev. */ bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */ bool attach_tracing_prog; /* true if tracing another tracing program */ + bool tramp_linked; /* true if it was ever linked to a trampoline */ bool func_proto_unreliable; bool tail_call_reachable; bool xdp_has_frags; diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 36b65797877d..b83ec99f1a13 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -45,18 +45,20 @@ struct bpf_reg_state { union { /* valid when type == PTR_TO_PACKET */ int range; - - /* valid when type == CONST_PTR_TO_MAP | PTR_TO_MAP_VALUE | - * PTR_TO_MAP_VALUE_OR_NULL + /* + * Valid when type == PTR_TO_STACK. Inside the callee two registers + * can be both PTR_TO_STACK like R1=fp-8 and R2=fp-8, but one of them + * points to this function stack while another to the caller's stack. + * To differentiate them 'frameno' is used which is an index in + * bpf_verifier_state->frame[] array pointing to bpf_func_state. */ - struct { - struct bpf_map *map_ptr; - /* To distinguish map lookups from outer map - * the map_uid is non-zero for registers - * pointing to inner maps. - */ - u32 map_uid; - }; + u8 frameno; + + /* + * For CONST_PTR_TO_MAP, PTR_TO_MAP_KEY, PTR_TO_MAP_VALUE and + * PTR_TO_INSN. + */ + struct bpf_map *map_ptr; /* for PTR_TO_BTF_ID */ struct { @@ -155,13 +157,12 @@ struct bpf_reg_state { * gets parent_id set to the dynptr's id. */ u32 parent_id; - /* Inside the callee two registers can be both PTR_TO_STACK like - * R1=fp-8 and R2=fp-8, but one of them points to this function stack - * while another to the caller's stack. To differentiate them 'frameno' - * is used which is an index in bpf_verifier_state->frame[] array - * pointing to bpf_func_state. + /* + * Distinguishes inner-map lookups and their keys and values. Zero for + * other registers. Kept outside the metadata union for ID remapping + * during state comparisons. */ - u32 frameno; + u32 map_uid; /* if (!precise && SCALAR_VALUE) min/max/tnum don't affect safety */ bool precise; }; @@ -679,6 +680,7 @@ struct bpf_insn_aux_data { bool nospec_result; /* result is unsafe under speculation, nospec must follow */ bool zext_dst; /* this insn zero extends dst reg */ bool needs_zext; /* alu op needs to clear upper bits */ + bool prevent_zext; /* alu op cannot be zext (already used with 64-bit scalars) */ bool non_sleepable; /* helper/kfunc may be called from non-sleepable context */ bool is_iter_next; /* bpf_iter__next() kfunc call */ bool call_with_percpu_alloc_ptr; /* {this,per}_cpu_ptr() with prog percpu alloc */ @@ -1197,6 +1199,8 @@ static inline void bpf_trampoline_unpack_key(u64 key, u32 *obj_id, u32 *btf_id) int bpf_prepare_btf_info(struct bpf_verifier_env *env, const union bpf_attr *attr, bpfptr_t uattr); +int bpf_check_core_relo(struct bpf_verifier_env *env, + const union bpf_attr *attr, bpfptr_t uattr); int bpf_check_btf_info(struct bpf_verifier_env *env, const union bpf_attr *attr, bpfptr_t uattr); @@ -1236,11 +1240,18 @@ static inline int bpf_get_spi(s32 off) return (-off - 1) / BPF_REG_SIZE; } +/* + * Return the function state a stack pointer register refers to. frameno + * shares storage with other pointer metadata, so return NULL for any + * other register type instead of indexing frame[] with aliased bytes. + */ static inline struct bpf_func_state *bpf_func(struct bpf_verifier_env *env, const struct bpf_reg_state *reg) { struct bpf_verifier_state *cur = env->cur_state; + if (reg->type != PTR_TO_STACK) + return NULL; return cur->frame[reg->frameno]; } diff --git a/include/linux/btf.h b/include/linux/btf.h index 89d5a5c4f117..7c62ea17b116 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -80,6 +80,7 @@ #define KF_ARENA_ARG2 (1 << 15) /* kfunc takes an arena pointer as its second argument */ #define KF_IMPLICIT_ARGS (1 << 16) /* kfunc has implicit arguments supplied by the verifier */ #define KF_SPINLOCK_SAFE (1 << 17) /* kfunc is allowed inside bpf_spin_lock-ed region */ +#define KF_PERFMON (1 << 18) /* kfunc requires CAP_PERFMON */ /* * Tag marking a kernel function as a kfunc. This is meant to minimize the diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 35ce665edfd8..d3d0d9d02aa4 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -420,6 +420,9 @@ struct cpufreq_driver { /* Will be called after the driver is fully initialized */ void (*ready)(struct cpufreq_policy *policy); + /* Return the capacity reference frequency for policy. */ + unsigned int (*scale_freq_ref)(struct cpufreq_policy *policy); + struct freq_attr **attr; /* platform specific boost support code */ diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index 253600c0eccd..c4c9ce038611 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -944,6 +944,7 @@ struct kernel_ethtool_ts_info { #define ETHTOOL_OP_NEEDS_RTNL_SPAUSEPARAM BIT(6) #define ETHTOOL_OP_NEEDS_RTNL_RSS BIT(7) #define ETHTOOL_OP_NEEDS_RTNL_GLINK BIT(8) +#define ETHTOOL_OP_NEEDS_RTNL_TEST BIT(9) /** * struct ethtool_ops - optional netdev operations @@ -981,6 +982,7 @@ struct kernel_ethtool_ts_info { * - netdev_update_features() * - netif_set_real_num_tx_queues() * - ethtool_op_get_link() (syncs link watch under rtnl_lock) + * - netif_open() / netif_close() (used by @self_test) * * @get_drvinfo: Report driver/device information. Modern drivers no * longer have to implement this callback. Most fields are diff --git a/include/linux/hrtimer_rearm.h b/include/linux/hrtimer_rearm.h index 17a81826bd9a..12e42721c8e0 100644 --- a/include/linux/hrtimer_rearm.h +++ b/include/linux/hrtimer_rearm.h @@ -45,7 +45,7 @@ hrtimer_rearm_deferred_user_irq(unsigned long *tif_work, const unsigned long tif clear_thread_flag(TIF_HRTIMER_REARM); __hrtimer_rearm_deferred(); /* Don't go into the loop if HRTIMER_REARM was the only flag */ - *tif_work &= ~TIF_HRTIMER_REARM; + *tif_work &= ~_TIF_HRTIMER_REARM; return !*tif_work; } return false; diff --git a/include/linux/if_vlan.h b/include/linux/if_vlan.h index 20cc16ea4e5a..4846032bf4ff 100644 --- a/include/linux/if_vlan.h +++ b/include/linux/if_vlan.h @@ -365,6 +365,9 @@ static inline int __vlan_insert_inner_tag(struct sk_buff *skb, const u8 meta_len = mac_len > ETH_TLEN ? skb_metadata_len(skb) : 0; struct vlan_ethhdr *veth; + if (unlikely(!pskb_may_pull(skb, mac_len))) + return -EINVAL; + if (skb_cow_head(skb, meta_len + VLAN_HLEN) < 0) return -ENOMEM; diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 03bfc92864b6..c844def6dd95 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -791,7 +791,6 @@ struct kvm { /* The current active memslot set for each address space */ struct kvm_memslots __rcu *memslots[KVM_MAX_NR_ADDRESS_SPACES]; struct xarray vcpu_array; - DECLARE_BITMAP(vcpu_ids, KVM_MAX_VCPU_IDS); /* * Protected by slots_lock, but can be read outside if an * incorrect answer is acceptable. @@ -2324,13 +2323,18 @@ static inline bool kvm_test_request(int req, struct kvm_vcpu *vcpu) return test_bit(req & KVM_REQUEST_MASK, (void *)&vcpu->requests); } -static inline void kvm_clear_request(int req, struct kvm_vcpu *vcpu) +static __always_inline void kvm_clear_request(int req, struct kvm_vcpu *vcpu) { + BUILD_BUG_ON(req == KVM_REQ_VM_DEAD); + clear_bit(req & KVM_REQUEST_MASK, (void *)&vcpu->requests); } -static inline bool kvm_check_request(int req, struct kvm_vcpu *vcpu) +static __always_inline bool kvm_check_request(int req, struct kvm_vcpu *vcpu) { + /* Once a VM is dead, it needs to stay dead. */ + BUILD_BUG_ON(req == KVM_REQ_VM_DEAD); + if (kvm_test_request(req, vcpu)) { kvm_clear_request(req, vcpu); diff --git a/include/linux/libata.h b/include/linux/libata.h index 313e96173b19..48bde275968d 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -76,7 +76,7 @@ enum ata_quirks { __ATA_QUIRK_MAX_SEC, /* Limit max sectors */ __ATA_QUIRK_MAX_TRIM_128M, /* Limit max trim size to 128M */ __ATA_QUIRK_NO_NCQ_ON_ATI, /* Disable NCQ on ATI chipset */ - __ATA_QUIRK_NO_LPM_ON_ATI, /* Disable LPM on ATI chipset */ + __ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD, /* Disable LPM on ATI and AMD chipsets */ __ATA_QUIRK_NO_ID_DEV_LOG, /* Identify device log missing */ __ATA_QUIRK_NO_LOG_DIR, /* Do not read log directory */ __ATA_QUIRK_NO_FUA, /* Do not use FUA */ @@ -115,7 +115,7 @@ enum { ATA_QUIRK_MAX_SEC = BIT_ULL(__ATA_QUIRK_MAX_SEC), ATA_QUIRK_MAX_TRIM_128M = BIT_ULL(__ATA_QUIRK_MAX_TRIM_128M), ATA_QUIRK_NO_NCQ_ON_ATI = BIT_ULL(__ATA_QUIRK_NO_NCQ_ON_ATI), - ATA_QUIRK_NO_LPM_ON_ATI = BIT_ULL(__ATA_QUIRK_NO_LPM_ON_ATI), + ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD = BIT_ULL(__ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD), ATA_QUIRK_NO_ID_DEV_LOG = BIT_ULL(__ATA_QUIRK_NO_ID_DEV_LOG), ATA_QUIRK_NO_LOG_DIR = BIT_ULL(__ATA_QUIRK_NO_LOG_DIR), ATA_QUIRK_NO_FUA = BIT_ULL(__ATA_QUIRK_NO_FUA), diff --git a/include/linux/mempool.h b/include/linux/mempool.h index a0fa6d43e0dc..6da502aef2f7 100644 --- a/include/linux/mempool.h +++ b/include/linux/mempool.h @@ -70,6 +70,13 @@ int mempool_alloc_bulk_noprof(struct mempool *pool, void **elem, #define mempool_alloc_bulk(...) \ alloc_hooks(mempool_alloc_bulk_noprof(__VA_ARGS__)) +/* + * Allocate a new element without dipping into the pool's reserves or + * waiting. Returns NULL on failure. + */ +#define mempool_alloc_noreserve(_pool, _gfp) \ + alloc_hooks((_pool)->alloc(_gfp, (_pool)->pool_data)) + void *mempool_alloc_preallocated(struct mempool *pool) __malloc; void mempool_free(void *element, struct mempool *pool); unsigned int mempool_free_bulk(struct mempool *pool, void **elem, diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 6d815f6440c9..f3e5a2fadbe5 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -1226,7 +1226,7 @@ struct mm_struct { struct mm_mm_cid mm_cid; /* sched_cache related statistics */ - struct sched_cache_stat sc_stat; + struct sched_cache_group *sched_cache_grp; #ifdef CONFIG_MMU atomic_long_t pgtables_bytes; /* size of all page tables */ #endif @@ -1624,8 +1624,9 @@ static inline unsigned int mm_cid_size(void) #endif /* CONFIG_SCHED_MM_CID */ #ifdef CONFIG_SCHED_CACHE -void mm_init_sched(struct mm_struct *mm, - struct sched_cache_time __percpu *pcpu_sched); +int mm_init_sched(struct mm_struct *mm, + struct sched_cache_time __percpu *pcpu_sched); +void mm_destroy_sched(struct mm_struct *mm); static inline int mm_alloc_sched_noprof(struct mm_struct *mm) { @@ -1635,17 +1636,11 @@ static inline int mm_alloc_sched_noprof(struct mm_struct *mm) if (!pcpu_sched) return -ENOMEM; - mm_init_sched(mm, pcpu_sched); - return 0; + return mm_init_sched(mm, pcpu_sched); } #define mm_alloc_sched(...) alloc_hooks(mm_alloc_sched_noprof(__VA_ARGS__)) -static inline void mm_destroy_sched(struct mm_struct *mm) -{ - free_percpu(mm->sc_stat.pcpu_sched); - mm->sc_stat.pcpu_sched = NULL; -} #else /* !CONFIG_SCHED_CACHE */ static inline int mm_alloc_sched(struct mm_struct *mm) { return 0; } diff --git a/include/linux/mtd/nand.h b/include/linux/mtd/nand.h index 09c8c93e4dba..6936180b6ea5 100644 --- a/include/linux/mtd/nand.h +++ b/include/linux/mtd/nand.h @@ -305,6 +305,8 @@ int nand_ecc_prepare_io_req(struct nand_device *nand, struct nand_page_io_req *req); int nand_ecc_finish_io_req(struct nand_device *nand, struct nand_page_io_req *req); +bool nand_ecc_is_pipelined(const struct nand_device *nand); + bool nand_ecc_is_strong_enough(struct nand_device *nand); #if IS_REACHABLE(CONFIG_MTD_NAND_CORE) diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 87cafc932e9e..3cff2174dc03 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -887,6 +887,7 @@ enum net_device_path_type { DEV_PATH_DSA, DEV_PATH_MTK_WDMA, DEV_PATH_TUN, + DEV_PATH_IEEE80211, }; struct net_device_path { @@ -953,6 +954,8 @@ struct net_device_path_ctx { u16 id; __be16 proto; } vlan[NET_DEVICE_PATH_VLAN_MAX]; + + bool ieee80211; }; enum tc_setup_type { diff --git a/include/linux/netfilter_netdev.h b/include/linux/netfilter_netdev.h index 3175073a66ba..2a854a0bbd4b 100644 --- a/include/linux/netfilter_netdev.h +++ b/include/linux/netfilter_netdev.h @@ -133,7 +133,7 @@ static inline struct sk_buff *nf_hook_egress(struct sk_buff *skb, int *rc, static inline void nf_skip_egress(struct sk_buff *skb, bool skip) { -#ifdef CONFIG_NETFILTER_SKIP_EGRESS +#ifdef CONFIG_NET_EGRESS skb->nf_skip_egress = skip; #endif } diff --git a/include/linux/netfs.h b/include/linux/netfs.h index b4dd32863dd4..fe6275e55fc2 100644 --- a/include/linux/netfs.h +++ b/include/linux/netfs.h @@ -397,6 +397,8 @@ ssize_t netfs_unbuffered_write_iter(struct kiocb *iocb, struct iov_iter *from); ssize_t netfs_unbuffered_write_iter_locked(struct kiocb *iocb, struct iov_iter *iter, struct netfs_group *netfs_group); ssize_t netfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from); +void netfs_clear_stale_post_isize(struct inode *inode, uoff_t from, + uoff_t to); /* Single, monolithic object read/write API. */ void netfs_single_mark_inode_dirty(struct inode *inode); diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 5842552294c1..2d5e7687875b 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1467,23 +1467,6 @@ static inline u32 perf_sample_data_size(struct perf_sample_data *data, return size; } -/* - * Clear all bitfields in the perf_branch_entry. - * The to and from fields are not cleared because they are - * systematically modified by caller. - */ -static inline void perf_clear_branch_entry_bitfields(struct perf_branch_entry *br) -{ - br->mispred = 0; - br->predicted = 0; - br->in_tx = 0; - br->abort = 0; - br->cycles = 0; - br->type = 0; - br->spec = PERF_BR_SPEC_NA; - br->reserved = 0; -} - extern void perf_output_sample(struct perf_output_handle *handle, struct perf_event_header *header, struct perf_sample_data *data, @@ -1523,9 +1506,10 @@ is_default_overflow_handler(struct perf_event *event) } extern void -perf_event_header__init_id(struct perf_event_header *header, - struct perf_sample_data *data, - struct perf_event *event); +perf_event_header__init(struct perf_event_header *header, + struct perf_sample_data *data, + u32 type, u16 misc, u16 size, + struct perf_event *event); extern void perf_event__output_id_sample(struct perf_event *event, struct perf_output_handle *handle, diff --git a/include/linux/rtnetlink.h b/include/linux/rtnetlink.h index 95729339e7a5..a54ec40d095c 100644 --- a/include/linux/rtnetlink.h +++ b/include/linux/rtnetlink.h @@ -186,7 +186,12 @@ void net_dec_ingress_queue(void); #ifdef CONFIG_NET_EGRESS void net_inc_egress_queue(void); void net_dec_egress_queue(void); -void netdev_xmit_skip_txqueue(bool skip); +bool netdev_xmit_skip_txqueue(bool skip); +#else +static inline bool netdev_xmit_skip_txqueue(bool skip) +{ + return false; +} #endif void rtnetlink_init(void); diff --git a/include/linux/sched.h b/include/linux/sched.h index 705970d07614..d35ae49a991f 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1433,6 +1433,7 @@ struct task_struct { #ifdef CONFIG_SCHED_CACHE struct callback_head cache_work; + struct sched_cache_group __rcu *sched_cache_grp; int preferred_llc; /* 1: task was enqueued to its preferred LLC, 0 otherwise */ int pref_llc_queued; @@ -2405,7 +2406,7 @@ struct sched_cache_time { unsigned long epoch; }; -struct sched_cache_stat { +struct sched_cache_group { struct sched_cache_time __percpu *pcpu_sched; raw_spinlock_t lock; unsigned long epoch; @@ -2413,11 +2414,26 @@ struct sched_cache_stat { unsigned long next_scan; unsigned long footprint; int cpu; + refcount_t refcnt; + struct rcu_head rcu; } ____cacheline_aligned_in_smp; +struct sched_cache_group *sched_cache_group_get(struct sched_cache_group *grp); +struct sched_cache_group *task_cache_group_get(struct task_struct *p); + +void sched_cache_fork(struct task_struct *p); +void sched_cache_fork_cleanup(struct task_struct *p); +void sched_cache_exec_mmap(struct task_struct *p, struct mm_struct *mm); +void sched_cache_exit_mm(struct task_struct *p); + #else -struct sched_cache_stat { }; +struct sched_cache_group { }; + +static inline void sched_cache_fork(struct task_struct *p) { } +static inline void sched_cache_fork_cleanup(struct task_struct *p) { } +static inline void sched_cache_exec_mmap(struct task_struct *p, struct mm_struct *mm) { } +static inline void sched_cache_exit_mm(struct task_struct *p) { } #endif diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h index b5d9d7c2b8ad..f96812d71c51 100644 --- a/include/linux/sched/topology.h +++ b/include/linux/sched/topology.h @@ -281,9 +281,9 @@ static inline int task_node(const struct task_struct *p) } #ifdef CONFIG_SCHED_CACHE -extern void sched_update_llc_bytes(unsigned int cpu); +extern void sched_update_llc_bytes(const struct cpumask *cpus); #else -static inline void sched_update_llc_bytes(unsigned int cpu) { } +static inline void sched_update_llc_bytes(const struct cpumask *cpus) { } #endif #endif /* _LINUX_SCHED_TOPOLOGY_H */ diff --git a/include/linux/siphash.h b/include/linux/siphash.h index 9153e77382e1..00c58bbe549a 100644 --- a/include/linux/siphash.h +++ b/include/linux/siphash.h @@ -75,8 +75,10 @@ static inline u64 ___siphash_aligned(const __le64 *data, size_t len, /** * siphash - compute 64-bit siphash PRF value * @data: buffer to hash - * @size: size of @data + * @len: size of @data * @key: the siphash key + * + * Returns: siphash PRF value */ static inline u64 siphash(const void *data, size_t len, const siphash_key_t *key) @@ -125,8 +127,10 @@ static inline u32 ___hsiphash_aligned(const __le32 *data, size_t len, /** * hsiphash - compute 32-bit hsiphash PRF value * @data: buffer to hash - * @size: size of @data + * @len: size of @data * @key: the hsiphash key + * + * Returns: hsiphash PRF value */ static inline u32 hsiphash(const void *data, size_t len, const hsiphash_key_t *key) diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 421f6fc45451..a3ff380d43f0 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1020,7 +1020,7 @@ struct sk_buff { #ifdef CONFIG_NET_REDIRECT __u8 from_ingress:1; #endif -#ifdef CONFIG_NETFILTER_SKIP_EGRESS +#ifdef CONFIG_NET_EGRESS __u8 nf_skip_egress:1; #endif #ifdef CONFIG_SKB_DECRYPTED @@ -1834,22 +1834,6 @@ static inline void skb_zcopy_set(struct sk_buff *skb, struct ubuf_info *uarg, } } -static inline void skb_zcopy_set_nouarg(struct sk_buff *skb, void *val) -{ - skb_shinfo(skb)->destructor_arg = (void *)((uintptr_t) val | 0x1UL); - skb_shinfo(skb)->flags |= SKBFL_ZEROCOPY_FRAG; -} - -static inline bool skb_zcopy_is_nouarg(struct sk_buff *skb) -{ - return (uintptr_t) skb_shinfo(skb)->destructor_arg & 0x1UL; -} - -static inline void *skb_zcopy_get_nouarg(struct sk_buff *skb) -{ - return (void *)((uintptr_t) skb_shinfo(skb)->destructor_arg & ~0x1UL); -} - static inline void net_zcopy_put(struct ubuf_info *uarg) { if (uarg) @@ -1872,8 +1856,7 @@ static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy_success) struct ubuf_info *uarg = skb_zcopy(skb); if (uarg) { - if (!skb_zcopy_is_nouarg(skb)) - uarg->ops->complete(skb, uarg, zerocopy_success); + uarg->ops->complete(skb, uarg, zerocopy_success); skb_shinfo(skb)->flags &= ~SKBFL_ALL_ZEROCOPY; } @@ -4372,7 +4355,10 @@ skb_header_pointer_careful(const struct sk_buff *skb, int offset, static inline void * __must_check skb_pointer_if_linear(const struct sk_buff *skb, int offset, int len) { - if (likely(skb_headlen(skb) - offset >= len)) + unsigned int uoffset = (unsigned int)offset; + + if (likely(uoffset <= skb_headlen(skb) && + (unsigned int)len <= skb_headlen(skb) - uoffset)) return skb->data + offset; return NULL; } diff --git a/include/linux/soc/qcom/geni-se.h b/include/linux/soc/qcom/geni-se.h index 29a53bbc0dd4..5f18d281e6a4 100644 --- a/include/linux/soc/qcom/geni-se.h +++ b/include/linux/soc/qcom/geni-se.h @@ -347,17 +347,12 @@ struct geni_se { #define QUP_SE_VERSION_2_5 0x20050000 /* - * Define bandwidth thresholds that cause the underlying Core 2X interconnect - * clock to run at the named frequency. These baseline values are recommended - * by the hardware team, and are not dynamically scaled with GENI bandwidth - * beyond basic on/off. + * QUP Core 2X clock votes used by GENI clients through the "qup-core" ICC + * path. Values are in Bps and must be converted with Bps_to_icc() before + * setting avg_bw. */ -#define CORE_2X_19_2_MHZ 960 -#define CORE_2X_50_MHZ 2500 -#define CORE_2X_100_MHZ 5000 -#define CORE_2X_150_MHZ 7500 -#define CORE_2X_200_MHZ 10000 -#define CORE_2X_236_MHZ 16383 +#define CORE_2X_19_2_MHZ 9600000 +#define CORE_2X_50_MHZ 25000000 #define GENI_DEFAULT_BW Bps_to_icc(1000) diff --git a/include/linux/thunderbolt.h b/include/linux/thunderbolt.h index d48623fda79b..b62dfa52b149 100644 --- a/include/linux/thunderbolt.h +++ b/include/linux/thunderbolt.h @@ -22,6 +22,7 @@ struct device; #include #include #include +#include #include #include #include @@ -565,6 +566,7 @@ struct tb_nhi { * @interval_nsec: Interval counter if interrupt throttling is to be * used with this ring (in ns) * @wait: Used to signal that the ring may be empty now + * @lock_key: Lock validator class key per-ring */ struct tb_ring { spinlock_t lock; @@ -590,6 +592,7 @@ struct tb_ring { void *poll_data; unsigned int interval_nsec; wait_queue_head_t wait; + struct lock_class_key lock_key; }; /* Leave ring interrupt enabled on suspend */ diff --git a/include/linux/tty.h b/include/linux/tty.h index 0a46e4054dec..833bb7b97ceb 100644 --- a/include/linux/tty.h +++ b/include/linux/tty.h @@ -168,6 +168,7 @@ struct tty_operations; * @write_wait: concurrent writers are waiting in this queue until they are * allowed to write * @read_wait: readers wait for data in this queue + * @break_wait: wait queue for timed breaks * @hangup_work: normally a work to perform a hangup (do_tty_hangup()); while * freeing the tty, (re)used to release_one_tty() * @disc_data: pointer to @ldisc's private data (e.g. to &struct n_tty_data) @@ -230,6 +231,7 @@ struct tty_struct { struct fasync_struct *fasync; wait_queue_head_t write_wait; wait_queue_head_t read_wait; + wait_queue_head_t break_wait; struct work_struct hangup_work; void *disc_data; void *driver_data; diff --git a/include/linux/tty_driver.h b/include/linux/tty_driver.h index 1f2896e56e77..40ae3bb13e11 100644 --- a/include/linux/tty_driver.h +++ b/include/linux/tty_driver.h @@ -73,6 +73,11 @@ struct serial_struct; * @TTY_DRIVER_NO_WORKQUEUE: * Do not create workqueue when tty_register_driver(). Whenever set, flip * buffer workqueue can be set by tty_port_link_wq() for every port. + * + * @TTY_DRIVER_RESET_SAVED_TERMIOS: + * Reset any saved termios settings on device registration when reusing a + * minor number. Must only be set by drivers that guarantee that the minor + * number is no longer in use. */ enum tty_driver_flag { TTY_DRIVER_INSTALLED = BIT(0), @@ -84,6 +89,7 @@ enum tty_driver_flag { TTY_DRIVER_DYNAMIC_ALLOC = BIT(6), TTY_DRIVER_UNNUMBERED_NODE = BIT(7), TTY_DRIVER_NO_WORKQUEUE = BIT(8), + TTY_DRIVER_RESET_SAVED_TERMIOS = BIT(9), }; enum tty_driver_type { diff --git a/include/linux/tty_port.h b/include/linux/tty_port.h index 23cad403bb8f..8d22c59c6f15 100644 --- a/include/linux/tty_port.h +++ b/include/linux/tty_port.h @@ -245,7 +245,8 @@ bool tty_port_carrier_raised(struct tty_port *port); void tty_port_raise_dtr_rts(struct tty_port *port); void tty_port_lower_dtr_rts(struct tty_port *port); void tty_port_hangup(struct tty_port *port); -void __tty_port_tty_hangup(struct tty_port *port, bool check_clocal, bool async); +void tty_port_tty_hangup(struct tty_port *port, bool check_clocal); +void tty_port_tty_vhangup(struct tty_port *port); void tty_port_tty_wakeup(struct tty_port *port); int tty_port_block_til_ready(struct tty_port *port, struct tty_struct *tty, struct file *filp); @@ -264,25 +265,6 @@ static inline int tty_port_users(struct tty_port *port) return port->count + port->blocked_open; } -/** - * tty_port_tty_hangup - helper to hang up a tty asynchronously - * @port: tty port - * @check_clocal: hang only ttys with %CLOCAL unset? - */ -static inline void tty_port_tty_hangup(struct tty_port *port, bool check_clocal) -{ - __tty_port_tty_hangup(port, check_clocal, true); -} - -/** - * tty_port_tty_vhangup - helper to hang up a tty synchronously - * @port: tty port - */ -static inline void tty_port_tty_vhangup(struct tty_port *port) -{ - __tty_port_tty_hangup(port, false, false); -} - #ifdef CONFIG_TTY void tty_kref_put(struct tty_struct *tty); __DEFINE_CLASS_IS_CONDITIONAL(tty_port_tty, true); diff --git a/include/linux/usb/serial.h b/include/linux/usb/serial.h index 534e6650e2aa..ab0a32a90629 100644 --- a/include/linux/usb/serial.h +++ b/include/linux/usb/serial.h @@ -381,6 +381,7 @@ void usb_serial_handle_dcd_change(struct usb_serial_port *usb_port, int usb_serial_bus_register(struct usb_serial_driver *device); void usb_serial_bus_deregister(struct usb_serial_driver *device); +void usb_serial_bus_remove_new_id(struct usb_serial_driver *driver); extern const struct bus_type usb_serial_bus_type; extern struct tty_driver *usb_serial_tty_driver; diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h index 4105c446ca98..fa2a367731b5 100644 --- a/include/net/bluetooth/hci_core.h +++ b/include/net/bluetooth/hci_core.h @@ -62,7 +62,7 @@ struct inquiry_entry { NAME_PENDING, NAME_KNOWN, } name_state; - __u32 timestamp; + unsigned long timestamp; struct inquiry_data data; }; @@ -78,7 +78,7 @@ struct discovery_state { struct list_head all; /* All devices found during inquiry */ struct list_head unknown; /* Name state not known */ struct list_head resolve; /* Name needs to be resolved */ - __u32 timestamp; + unsigned long timestamp; bdaddr_t last_adv_addr; u8 last_adv_addr_type; s8 last_adv_rssi; @@ -562,6 +562,7 @@ struct hci_dev { struct list_head link_keys; struct list_head long_term_keys; struct list_head identity_resolving_keys; + struct mutex remote_oob_lock; struct list_head remote_oob_data; struct list_head le_accept_list; struct list_head le_resolv_list; diff --git a/include/net/dst.h b/include/net/dst.h index 307073eae7f8..dbedfe72e1fd 100644 --- a/include/net/dst.h +++ b/include/net/dst.h @@ -455,7 +455,8 @@ static inline unsigned int dst_dev_overhead(struct dst_entry *dst, struct sk_buff *skb) { if (likely(dst)) - return LL_RESERVED_SPACE(dst->dev); + return max_t(unsigned int, skb->mac_len, + LL_RESERVED_SPACE(dst->dev)); return skb->mac_len; } diff --git a/include/net/gso.h b/include/net/gso.h index 29975440cad5..0749230d414e 100644 --- a/include/net/gso.h +++ b/include/net/gso.h @@ -19,10 +19,18 @@ struct skb_gso_cb { int encap_level; __wsum csum; __u16 csum_start; + /* Number of IPv4/IPv6 GSO handler entries for this packet. */ + u8 recursion_counter; }; #define SKB_GSO_CB_OFFSET 32 #define SKB_GSO_CB(skb) ((struct skb_gso_cb *)((skb)->cb + SKB_GSO_CB_OFFSET)) +static inline bool gso_recursion_inc_test(struct sk_buff *skb, + unsigned int limit) +{ + return ++SKB_GSO_CB(skb)->recursion_counter > limit; +} + static inline int skb_tnl_header_len(const struct sk_buff *inner_skb) { return (skb_mac_header(inner_skb) - inner_skb->head) - diff --git a/include/net/gue.h b/include/net/gue.h index caefd6da8693..d377155fd0b3 100644 --- a/include/net/gue.h +++ b/include/net/gue.h @@ -84,8 +84,9 @@ static inline size_t guehdr_priv_flags_len(__be32 flags) } /* Validate standard and private flags. Returns non-zero (meaning invalid) - * if there is an unknown standard or private flags, or the options length for - * the flags exceeds the options length specific in hlen of the GUE header. + * if there is an unknown standard or private flags, if the options length for + * the flags exceeds the options length specified in hlen of the GUE header, or + * if a private option contains invalid data. */ static inline int validate_gue_flags(struct guehdr *guehdr, size_t optlen) { @@ -103,8 +104,8 @@ static inline int validate_gue_flags(struct guehdr *guehdr, size_t optlen) /* Private flags are last four bytes accounted in * guehdr_flags_len */ - __be32 pflags = *(__be32 *)((void *)&guehdr[1] + - len - GUE_LEN_PRIV); + void *data = (void *)&guehdr[1] + len; + __be32 pflags = *(__be32 *)(data - GUE_LEN_PRIV); if (pflags & ~GUE_PFLAGS_ALL) return 1; @@ -112,6 +113,16 @@ static inline int validate_gue_flags(struct guehdr *guehdr, size_t optlen) len += guehdr_priv_flags_len(pflags); if (len > optlen) return 1; + + if (pflags & GUE_PFLAG_REMCSUM) { + __be16 *pd = data; + + /* The field offset pd[1] must not be less + * than the start pd[0]. + */ + if (ntohs(pd[1]) < ntohs(pd[0])) + return 1; + } } return 0; diff --git a/include/net/ip6_route.h b/include/net/ip6_route.h index b9e8d2b759e9..0f9b7a260d25 100644 --- a/include/net/ip6_route.h +++ b/include/net/ip6_route.h @@ -101,12 +101,12 @@ static inline struct dst_entry *ip6_route_output(struct net *net, } /* Only conditionally release dst if flags indicates - * !RT6_LOOKUP_F_DST_NOREF or dst is in uncached_list. + * !RT6_LOOKUP_F_DST_NOREF or dst is uncached. */ static inline void ip6_rt_put_flags(struct rt6_info *rt, int flags) { if (!(flags & RT6_LOOKUP_F_DST_NOREF) || - !list_empty(&rt->dst.rt_uncached)) + rt->dst.rt_uncached_list) ip6_rt_put(rt); } diff --git a/include/net/netfilter/nf_conntrack.h b/include/net/netfilter/nf_conntrack.h index bc42dd0e10e6..c39425e54d87 100644 --- a/include/net/netfilter/nf_conntrack.h +++ b/include/net/netfilter/nf_conntrack.h @@ -185,6 +185,11 @@ static inline void nf_ct_put(struct nf_conn *ct) nf_ct_destroy(&ct->ct_general); } +static inline bool nf_ct_shared(const struct nf_conn *ct) +{ + return refcount_read(&ct->ct_general.use) > 1; +} + /* load module; enable/disable conntrack in this namespace */ int nf_ct_netns_get(struct net *net, u8 nfproto); void nf_ct_netns_put(struct net *net, u8 nfproto); diff --git a/include/net/netfilter/nf_flow_table.h b/include/net/netfilter/nf_flow_table.h index f2e2771f188f..5b611efaa3cd 100644 --- a/include/net/netfilter/nf_flow_table.h +++ b/include/net/netfilter/nf_flow_table.h @@ -183,10 +183,10 @@ enum nf_flow_flags { NF_FLOW_DNAT, NF_FLOW_CLOSING, NF_FLOW_TEARDOWN, + NF_FLOW_PENDING, NF_FLOW_HW, NF_FLOW_HW_DYING, NF_FLOW_HW_DEAD, - NF_FLOW_HW_PENDING, NF_FLOW_HW_BIDIRECTIONAL, NF_FLOW_HW_ESTABLISHED, }; diff --git a/include/net/nfc/hci.h b/include/net/nfc/hci.h index 756c11084f65..86ed63e5d533 100644 --- a/include/net/nfc/hci.h +++ b/include/net/nfc/hci.h @@ -144,7 +144,7 @@ struct nfc_hci_dev { data_exchange_cb_t async_cb; void *async_cb_context; - u8 *gb; + u8 gb[NFC_MAX_GT_LEN]; size_t gb_len; unsigned long quirks; diff --git a/include/net/nfc/nfc.h b/include/net/nfc/nfc.h index c54df042db6b..bcafab5c53e5 100644 --- a/include/net/nfc/nfc.h +++ b/include/net/nfc/nfc.h @@ -273,7 +273,8 @@ struct sk_buff *nfc_alloc_recv_skb(unsigned int size, gfp_t gfp); int nfc_set_remote_general_bytes(struct nfc_dev *dev, const u8 *gt, u8 gt_len); -u8 *nfc_get_local_general_bytes(struct nfc_dev *dev, size_t *gb_len); +u8 *nfc_get_local_general_bytes(struct nfc_dev *dev, u8 *out_gb, + size_t gb_max_len, size_t *gb_len); int nfc_fw_download_done(struct nfc_dev *dev, const char *firmware_name, u32 result); diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index f35bd06a6bad..1acaadb3e2cf 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -810,6 +810,15 @@ static inline bool skb_at_tc_ingress(const struct sk_buff *skb) #endif } +static inline bool skb_at_tc_egress(const struct sk_buff *skb) +{ +#ifdef CONFIG_NET_EGRESS + return skb->nf_skip_egress && !skb_at_tc_ingress(skb); +#else + return false; +#endif +} + static inline bool skb_skip_tc_classify(struct sk_buff *skb) { #ifdef CONFIG_NET_CLS_ACT diff --git a/include/net/tcp.h b/include/net/tcp.h index 436495ff2271..4416cdf9bf30 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -1232,9 +1232,9 @@ static inline bool tcp_skb_can_collapse_to(const struct sk_buff *skb) static inline bool tcp_skb_can_collapse(const struct sk_buff *to, const struct sk_buff *from) { - /* skb_cmp_decrypted() not needed, use tcp_write_collapse_fence() */ return likely(tcp_skb_can_collapse_to(to) && mptcp_skb_can_collapse(to, from) && + !skb_cmp_decrypted(to, from) && skb_pure_zcopy_same(to, from) && skb_frags_readable(to) == skb_frags_readable(from)); } @@ -2327,7 +2327,7 @@ static inline void tcp_rtx_queue_unlink_and_free(struct sk_buff *skb, struct soc static inline void tcp_write_collapse_fence(struct sock *sk) { - struct sk_buff *skb = tcp_write_queue_tail(sk); + struct sk_buff *skb = tcp_write_queue_tail(sk) ?: tcp_rtx_queue_tail(sk); if (skb) TCP_SKB_CB(skb)->eor = 1; diff --git a/include/net/xdp.h b/include/net/xdp.h index aa742f413c35..07231adfb5f8 100644 --- a/include/net/xdp.h +++ b/include/net/xdp.h @@ -106,6 +106,11 @@ struct xdp_buff { }; }; +static __always_inline void xdp_reinit_buff(struct xdp_buff *xdp) +{ + xdp->flags = 0; +} + static __always_inline bool xdp_buff_has_frags(const struct xdp_buff *xdp) { return !!(xdp->flags & XDP_FLAGS_HAS_FRAGS); diff --git a/include/sound/core.h b/include/sound/core.h index 2ca24ac7e37f..15be947a730e 100644 --- a/include/sound/core.h +++ b/include/sound/core.h @@ -340,6 +340,15 @@ static inline void snd_card_unref(struct snd_card *card) DEFINE_FREE(snd_card_unref, struct snd_card *, if (_T) snd_card_unref(_T)) +/* + * For automatic error handling at probe time, assign the card like: + * struct snd_card *card __free(snd_card_free) = NULL; + * then create and process as usual. + * But, don't forget to clear to NULL at successful return for avoiding + * the unexpected cleanup! + */ +DEFINE_FREE(snd_card_free, struct snd_card *, if (_T) snd_card_free(_T)) + #define snd_card_set_dev(card, devptr) ((card)->dev = (devptr)) /* device.c */ diff --git a/include/trace/events/landlock.h b/include/trace/events/landlock.h index d05253afaf59..3a43638c9bc2 100644 --- a/include/trace/events/landlock.h +++ b/include/trace/events/landlock.h @@ -10,14 +10,19 @@ #if !defined(_TRACE_LANDLOCK_H) || defined(TRACE_HEADER_MULTI_READ) #define _TRACE_LANDLOCK_H +#include +#include #include +#include #include #include #include #include #include +enum landlock_request_type; struct dentry; +struct landlock_blockers; struct landlock_domain; struct landlock_hierarchy; struct landlock_rule; @@ -26,6 +31,12 @@ struct path; struct sock; struct task_struct; +static_assert(sizeof(access_mask_t) <= sizeof(u64)); + +TRACE_DEFINE_ENUM(LANDLOCK_REQUEST_FS_CHANGE_TOPOLOGY); +TRACE_DEFINE_ENUM(LANDLOCK_REQUEST_FS_ACCESS); +TRACE_DEFINE_ENUM(LANDLOCK_REQUEST_NET_ACCESS); + #ifdef CREATE_TRACE_POINTS /* About 6 KiB, leaving about 2 KiB for sibling helpers and fixed fields. */ @@ -108,7 +119,7 @@ __trace_print_untrusted_str(struct trace_seq *p, const char *src, size_t len) * Fills the dense per-domain-layer array layers (one access mask per layer, * indexed by level - 1) from rule's sparse layer stack, keeping only the * requested rights (access_request). Layers with no matching rule entry get - * a zero mask. Shared by the check_rule_fs and check_rule_net events. + * a zero mask. Shared by the check_rule_inode and check_rule_net_port events. * * rule->layers is sorted by ascending level, with levels in the domain's * [1, num_layers] range (see landlock_merge_ruleset()), so every entry maps @@ -182,6 +193,9 @@ static inline const char *__trace_landlock_print_layers( /* Maps a shared _LANDLOCK_*_NAMES entry to a __print_flags() pair. */ #define _LANDLOCK_NAME_ENTRY(mask, name) { mask, name } +#define _LANDLOCK_FS_BLOCKER_TYPE_NAMES \ + { LANDLOCK_REQUEST_FS_CHANGE_TOPOLOGY, "change_topology" } + /** * DOC: Landlock trace events * @@ -192,22 +206,22 @@ static inline const char *__trace_landlock_print_layers( * Decision context * ~~~~~~~~~~~~~~~~ * - * A denial event, together with the lifecycle events, exposes the full - * set of inputs the verdict consumed, so a consumer that tracked domain - * creation (landlock_create_ruleset, landlock_create_domain) can verify - * or reproduce the Landlock decision rather than merely observe it - * happened. In who/what/why terms: who is the denying domain (the domain - * field, always the subject that enforced the policy, never the current - * task), what is the operation and its object, and why is every other - * input the verdict weighed. + * A denial event identifies the domain whose policy denied the request, the + * Landlock operation and policy object that were checked, and the blocker or + * domain relationship responsible for the denial. When tracing starts with + * sandbox construction, ruleset and domain events provide the policy history + * needed to interpret these identifiers. The denying domain is the subject + * that enforced the policy, not necessarily current. Generic tracepoints can + * provide additional operational context. * * Lifecycle consistency * ~~~~~~~~~~~~~~~~~~~~~~ * - * Lifecycle events are balanced: a creation event always has a matching - * deallocation event and vice versa, so an eBPF program can model object - * lifetimes from the trace stream without reconciliation logic. A creation - * event fires while the object is still private to the calling thread + * Lifecycle emission is balanced: a creation event always has a matching + * deallocation event and vice versa. A consumer that observes an object's + * complete lifetime can model it from this pair; one that attaches late or + * loses exported records must reconcile incomplete state. A creation event + * fires while the object is still private to the calling thread * (landlock_create_ruleset fires before the ruleset's file descriptor is * installed, so it cannot race a concurrent :manpage:`close(2)`); if fd * installation later fails and the ruleset is freed, free_ruleset still @@ -243,8 +257,8 @@ static inline const char *__trace_landlock_print_layers( * Field encoding * ~~~~~~~~~~~~~~ * - * Fields that mirror the Landlock UAPI use the same C types and endianness - * (e.g. network ports are __u64 in host endianness, like + * Fields that mirror the Landlock UAPI preserve their widths and endianness + * (e.g. network ports are u64 in host endianness, like * landlock_net_port_attr.port). Per-event details, such as where a value * is byte-swapped, live in the field's own kdoc. * @@ -281,6 +295,16 @@ static inline const char *__trace_landlock_print_layers( * the two parties without kernel-internal state. The ID is a scalar * snapshot, not a live domain pointer that could dangle: an optional * relational referent is a scalar (0 sentinel), not a nullable pointer. + * Nonzero IDs are unique within one boot. For ptrace, same_exec instead + * describes the tracer, even for + * PTRACE_TRACEME, and may differ from the current task. + * + * Blocker fields + * ~~~~~~~~~~~~~~ + * + * The filesystem and network blocker arguments identify the request type + * and carry its final missing access subset when applicable. The type + * determines how to interpret the access value. */ /* @@ -319,8 +343,8 @@ TRACE_EVENT(landlock_create_ruleset, TP_ARGS(ruleset), TP_STRUCT__entry( - __field( __u64, ruleset_id ) - __field( __u32, ruleset_version ) + __field( u64, ruleset_id ) + __field( u64, ruleset_version ) __field( access_mask_t, handled_fs ) __field( access_mask_t, handled_net ) __field( access_mask_t, scoped ) @@ -334,7 +358,7 @@ TRACE_EVENT(landlock_create_ruleset, __entry->scoped = ruleset->handled_masks.scope; ), - TP_printk("ruleset=%llx.%u handled_fs=%s handled_net=%s scoped=%s", + TP_printk("ruleset=%llx.%llu handled_fs=%s handled_net=%s scoped=%s", __entry->ruleset_id, __entry->ruleset_version, __print_flags(__entry->handled_fs, "|", _LANDLOCK_ACCESS_FS_NAMES), __print_flags(__entry->handled_net, "|", _LANDLOCK_ACCESS_NET_NAMES), @@ -359,8 +383,8 @@ TRACE_EVENT(landlock_free_ruleset, TP_ARGS(ruleset), TP_STRUCT__entry( - __field( __u64, ruleset_id ) - __field( __u32, ruleset_version ) + __field( u64, ruleset_id ) + __field( u64, ruleset_version ) ), TP_fast_assign( @@ -368,17 +392,19 @@ TRACE_EVENT(landlock_free_ruleset, __entry->ruleset_version = ruleset->version; ), - TP_printk("ruleset=%llx.%u", + TP_printk("ruleset=%llx.%llu", __entry->ruleset_id, __entry->ruleset_version) ); /** - * landlock_add_rule_fs - Filesystem rule added to a ruleset + * landlock_add_rule_path_beneath - Path-beneath rule added to a ruleset * * @ruleset: Source ruleset (never NULL). - * @access_rights: Effective access mask stored in the rule, not the raw - * sys_landlock_add_rule() argument (unhandled rights - * added). + * @flags: Complete validated landlock_add_rule_flags value supplied by this + * successful call, not the rule's accumulated quiet state. + * @access_rights: Canonical per-call access mask passed to + * landlock_insert_rule() after normalization, not the raw + * sys_landlock_add_rule() argument or accumulated rule. * @path: Filesystem path for the rule (never NULL). * @pathname: Resolved absolute path string (never NULL; error placeholder * on resolution failure). @@ -387,17 +413,17 @@ TRACE_EVENT(landlock_free_ruleset, * the reported ruleset is a stable snapshot that no concurrent writer can * change. */ -TRACE_EVENT(landlock_add_rule_fs, +TRACE_EVENT(landlock_add_rule_path_beneath, - TP_PROTO(const struct landlock_ruleset *ruleset, - access_mask_t access_rights, const struct path *path, + TP_PROTO(const struct landlock_ruleset *ruleset, u32 flags, + u64 access_rights, const struct path *path, const char *pathname), - TP_ARGS(ruleset, access_rights, path, pathname), + TP_ARGS(ruleset, flags, access_rights, path, pathname), TP_STRUCT__entry( - __field( __u64, ruleset_id ) - __field( __u32, ruleset_version ) + __field( u64, ruleset_id ) + __field( u64, ruleset_version ) __field( access_mask_t, access_rights ) __field( dev_t, dev ) __field( ino_t, ino ) @@ -418,7 +444,7 @@ TRACE_EVENT(landlock_add_rule_fs, __assign_str(pathname); ), - TP_printk("ruleset=%llx.%u access_rights=%s dev=%u:%u ino=%lu path=%s", + TP_printk("ruleset=%llx.%llu access_rights=%s dev=%u:%u ino=%lu path=%s", __entry->ruleset_id, __entry->ruleset_version, __print_flags(__entry->access_rights, "|", _LANDLOCK_ACCESS_FS_NAMES), MAJOR(__entry->dev), MINOR(__entry->dev), __entry->ino, @@ -427,31 +453,33 @@ TRACE_EVENT(landlock_add_rule_fs, ); /** - * landlock_add_rule_net - Network port rule added to a ruleset + * landlock_add_rule_net_port - Network-port rule added to a ruleset * * @ruleset: Source ruleset (never NULL). - * @access_rights: Effective access mask stored in the rule, not the raw - * sys_landlock_add_rule() argument (unhandled rights - * added). - * @port: Network port, the landlock_net_port_attr.port UAPI value - * forwarded directly. + * @flags: Complete validated landlock_add_rule_flags value supplied by this + * successful call, not the rule's accumulated quiet state. + * @access_rights: Canonical per-call access mask passed to + * landlock_insert_rule() after normalization, not the raw + * sys_landlock_add_rule() argument or accumulated rule. + * @port: Network port in host endianness, forwarded directly from + * &landlock_net_port_attr.port. * * Emitted by sys_landlock_add_rule() under the modified ruleset's lock, so * the reported ruleset is a stable snapshot that no concurrent writer can * change. */ -TRACE_EVENT(landlock_add_rule_net, +TRACE_EVENT(landlock_add_rule_net_port, - TP_PROTO(const struct landlock_ruleset *ruleset, - access_mask_t access_rights, __u64 port), + TP_PROTO(const struct landlock_ruleset *ruleset, u32 flags, + u64 access_rights, u64 port), - TP_ARGS(ruleset, access_rights, port), + TP_ARGS(ruleset, flags, access_rights, port), TP_STRUCT__entry( - __field( __u64, ruleset_id ) - __field( __u32, ruleset_version ) + __field( u64, ruleset_id ) + __field( u64, ruleset_version ) __field( access_mask_t, access_rights ) - __field( __u64, port ) + __field( u64, port ) ), TP_fast_assign( @@ -462,7 +490,7 @@ TRACE_EVENT(landlock_add_rule_net, __entry->port = port; ), - TP_printk("ruleset=%llx.%u access_rights=%s port=%llu", + TP_printk("ruleset=%llx.%llu access_rights=%s port=%llu", __entry->ruleset_id, __entry->ruleset_version, __print_flags(__entry->access_rights, "|", _LANDLOCK_ACCESS_NET_NAMES), __entry->port) @@ -495,10 +523,10 @@ TRACE_EVENT(landlock_create_domain, TP_ARGS(domain, ruleset), TP_STRUCT__entry( - __field( __u64, domain_id ) - __field( __u64, parent_id ) - __field( __u64, ruleset_id ) - __field( __u32, ruleset_version ) + __field( u64, domain_id ) + __field( u64, parent_id ) + __field( u64, ruleset_id ) + __field( u64, ruleset_version ) ), TP_fast_assign( @@ -510,7 +538,7 @@ TRACE_EVENT(landlock_create_domain, __entry->ruleset_version = ruleset->version; ), - TP_printk("domain=%llx parent=%llx ruleset=%llx.%u", + TP_printk("domain=%llx parent=%llx ruleset=%llx.%llu", __entry->domain_id, __entry->parent_id, __entry->ruleset_id, __entry->ruleset_version) ); @@ -557,7 +585,7 @@ TRACE_EVENT(landlock_enforce_domain, TP_ARGS(domain, complete, process_wide, no_new_privs), TP_STRUCT__entry( - __field( __u64, domain_id ) + __field( u64, domain_id ) __field( bool, complete ) __field( bool, process_wide ) __field( bool, no_new_privs ) @@ -595,8 +623,8 @@ TRACE_EVENT(landlock_free_domain, TP_ARGS(hierarchy), TP_STRUCT__entry( - __field( __u64, domain_id ) - __field( __u64, denials ) + __field( u64, domain_id ) + __field( u64, denials ) ), TP_fast_assign( @@ -609,7 +637,7 @@ TRACE_EVENT(landlock_free_domain, ); /** - * landlock_check_rule_fs - Filesystem rule evaluated during access check + * landlock_check_rule_inode - Inode rule evaluated during access check * * @domain: Enforcing domain (never NULL). * @rule: Matching rule with per-layer access masks (never NULL). @@ -622,16 +650,16 @@ TRACE_EVENT(landlock_free_domain, * domain layer. See Documentation/trace/events-landlock.rst for how to * interpret it. */ -TRACE_EVENT(landlock_check_rule_fs, +TRACE_EVENT(landlock_check_rule_inode, TP_PROTO(const struct landlock_domain *domain, const struct landlock_rule *rule, - access_mask_t access_request, const struct dentry *dentry), + u64 access_request, const struct dentry *dentry), TP_ARGS(domain, rule, access_request, dentry), TP_STRUCT__entry( - __field( __u64, domain_id ) + __field( u64, domain_id ) __field( access_mask_t, access_request ) __field( dev_t, dev ) __field( ino_t, ino ) @@ -648,7 +676,8 @@ TRACE_EVENT(landlock_check_rule_fs, __trace_landlock_fill_layers(__get_dynamic_array(grants), __get_dynamic_array_len(grants) / sizeof(access_mask_t), - rule, access_request); + rule, + (access_mask_t)access_request); ), TP_printk("domain=%llx access_request=%s dev=%u:%u ino=%lu grants=%s", @@ -659,7 +688,7 @@ TRACE_EVENT(landlock_check_rule_fs, ); /** - * landlock_check_rule_net - Network port rule evaluated during access check + * landlock_check_rule_net_port - Network port rule evaluated * * @domain: Enforcing domain (never NULL). * @rule: Matching rule with per-layer access masks (never NULL). @@ -671,18 +700,18 @@ TRACE_EVENT(landlock_check_rule_fs, * layer. See Documentation/trace/events-landlock.rst for how to * interpret it. */ -TRACE_EVENT(landlock_check_rule_net, +TRACE_EVENT(landlock_check_rule_net_port, TP_PROTO(const struct landlock_domain *domain, const struct landlock_rule *rule, - access_mask_t access_request, __u64 port), + u64 access_request, u64 port), TP_ARGS(domain, rule, access_request, port), TP_STRUCT__entry( - __field( __u64, domain_id ) + __field( u64, domain_id ) __field( access_mask_t, access_request ) - __field( __u64, port ) + __field( u64, port ) __dynamic_array(access_mask_t, grants, domain->num_layers) ), @@ -695,7 +724,8 @@ TRACE_EVENT(landlock_check_rule_net, __trace_landlock_fill_layers(__get_dynamic_array(grants), __get_dynamic_array_len(grants) / sizeof(access_mask_t), - rule, access_request); + rule, + (access_mask_t)access_request); ), TP_printk("domain=%llx access_request=%s port=%llu grants=%s", @@ -712,8 +742,7 @@ TRACE_EVENT(landlock_check_rule_net, * domain field. * @same_exec: Whether the current task entered the denying domain itself. * @logged: The domain's audit-logging decision for this denial. - * @blockers: Access mask that was blocked (zero for a mount-topology - * change, whose only blocker is the operation itself). + * @blockers: Request type and final missing access subset (never NULL). * @path: Filesystem path that was denied (never NULL). * @pathname: Resolved path string (never NULL; an error placeholder on * resolution failure). @@ -723,16 +752,17 @@ TRACE_EVENT(landlock_check_rule_net, TRACE_EVENT(landlock_deny_access_fs, TP_PROTO(const struct landlock_hierarchy *hierarchy, bool same_exec, - bool logged, access_mask_t blockers, const struct path *path, - const char *pathname), + bool logged, const struct landlock_blockers *blockers, + const struct path *path, const char *pathname), TP_ARGS(hierarchy, same_exec, logged, blockers, path, pathname), TP_STRUCT__entry( - __field( __u64, domain_id ) + __field( u64, domain_id ) __field( bool, same_exec ) __field( bool, logged ) - __field( access_mask_t, blockers ) + __field( enum landlock_request_type, blockers_type ) + __field( access_mask_t, blockers_access ) __field( dev_t, dev ) __field( ino_t, ino ) __string( pathname, pathname ) @@ -744,7 +774,8 @@ TRACE_EVENT(landlock_deny_access_fs, __entry->domain_id = hierarchy->id; __entry->same_exec = same_exec; __entry->logged = logged; - __entry->blockers = blockers; + __entry->blockers_type = blockers->type; + __entry->blockers_access = blockers->access; __entry->dev = path->dentry->d_sb->s_dev; /* * A negative dentry has no backing inode, so mirror the @@ -756,12 +787,20 @@ TRACE_EVENT(landlock_deny_access_fs, TP_printk("domain=%llx same_exec=%d logged=%d blockers=%s dev=%u:%u ino=%lu path=%s", __entry->domain_id, __entry->same_exec, __entry->logged, - __print_flags(__entry->blockers, "|", _LANDLOCK_ACCESS_FS_NAMES), + __entry->blockers_type == LANDLOCK_REQUEST_FS_ACCESS ? + __print_flags(__entry->blockers_access, "|", _LANDLOCK_ACCESS_FS_NAMES) : + __print_symbolic(__entry->blockers_type, + _LANDLOCK_FS_BLOCKER_TYPE_NAMES), MAJOR(__entry->dev), MINOR(__entry->dev), __entry->ino, __trace_print_untrusted_str(p, __get_str(pathname), __get_dynamic_array_len(pathname) - 1)) ); +static_assert(offsetof(struct sockaddr_in, sin_port) == + offsetof(struct sockaddr_in6, sin6_port)); +static_assert(sizeof_field(struct sockaddr_in, sin_port) == + sizeof_field(struct sockaddr_in6, sin6_port)); + /** * landlock_deny_access_net - Network access denied * @@ -769,54 +808,69 @@ TRACE_EVENT(landlock_deny_access_fs, * domain field. * @same_exec: Whether the current task entered the denying domain itself. * @logged: The domain's audit-logging decision for this denial. - * @blockers: Access mask that was blocked. - * @sk: Socket object (never NULL), read without a socket lock, so its - * fields are a best-effort snapshot. The denied endpoint is not - * available: the hook runs before :manpage:`bind(2)` / - * :manpage:`connect(2)` sets the socket addresses. - * @sport: Source port in host endianness, set for bind denials (zero for - * an autobind/ephemeral port); zero for connect and send denials. - * @dport: Destination port in host endianness, set for connect and send - * denials; zero for bind denials, and also zero for a UDP send to - * an AF_UNSPEC address on an IPv6 socket (indistinguishable from a - * real destination port 0). The bind-vs-connect direction is - * given by @blockers, not by which port is set. + * @blockers: Request type and final missing access subset (never NULL). + * @sk: Socket object (never NULL), read without a socket lock, so its fields + * are a best-effort snapshot. + * @socket_family: Socket-family snapshot used by the verdict. + * @address: Authoritative address checked by the verdict (never NULL). + * The producer copies @addrlen bytes from the checked address and + * zeroes the remaining storage before emission. The + * &sockaddr_in.sin_port or &sockaddr_in6.sin6_port member, when + * present, remains in network endianness. + * @addrlen: Validated signed length of @address. * - * Emitted when a Landlock domain denies a network operation. - * - * The port fields are converted from the socket's network byte order to - * host endianness before emitting. + * Emitted when a Landlock domain denies a network operation. The blocker + * identifies whether the address is a bind or connect/send policy object. + * The flattened port field is converted from the checked address to host + * endianness, or is -1 when no port was checked. Zero is a valid checked + * port. */ TRACE_EVENT(landlock_deny_access_net, TP_PROTO(const struct landlock_hierarchy *hierarchy, bool same_exec, - bool logged, access_mask_t blockers, const struct sock *sk, - __u64 sport, __u64 dport), + bool logged, const struct landlock_blockers *blockers, + const struct sock *sk, u16 socket_family, + const struct sockaddr_storage *address, int addrlen), - TP_ARGS(hierarchy, same_exec, logged, blockers, sk, sport, dport), + TP_ARGS(hierarchy, same_exec, logged, blockers, sk, socket_family, + address, addrlen), TP_STRUCT__entry( - __field( __u64, domain_id ) + __field( u64, domain_id ) __field( bool, same_exec ) __field( bool, logged ) - __field( access_mask_t, blockers ) - __field( __u64, sport ) - __field( __u64, dport ) + __field( enum landlock_request_type, blockers_type ) + __field( access_mask_t, blockers_access ) + __field( s64, port ) ), TP_fast_assign( + const struct sockaddr *const addr = + (const struct sockaddr *)address; + const bool has_port = + addrlen >= (int)offsetofend(struct sockaddr_in, sin_port) && + (addr->sa_family == AF_INET || + addr->sa_family == AF_INET6 || + (addr->sa_family == AF_UNSPEC && + socket_family == AF_INET)); + __entry->domain_id = hierarchy->id; __entry->same_exec = same_exec; __entry->logged = logged; - __entry->blockers = blockers; - __entry->sport = sport; - __entry->dport = dport; + __entry->blockers_type = blockers->type; + __entry->blockers_access = blockers->access; + __entry->port = + has_port ? + ntohs(((const struct sockaddr_in *)addr)->sin_port) : + -1; ), - TP_printk("domain=%llx same_exec=%d logged=%d blockers=%s sport=%llu dport=%llu", - __entry->domain_id, __entry->same_exec, __entry->logged, - __print_flags(__entry->blockers, "|", _LANDLOCK_ACCESS_NET_NAMES), - __entry->sport, __entry->dport) + TP_printk("domain=%llx same_exec=%d logged=%d blockers=%s port=%lld", + __entry->domain_id, __entry->same_exec, __entry->logged, + __entry->blockers_type == LANDLOCK_REQUEST_NET_ACCESS ? + __print_flags(__entry->blockers_access, "|", _LANDLOCK_ACCESS_NET_NAMES) : + "unknown", + __entry->port) ); /** @@ -824,12 +878,14 @@ TRACE_EVENT(landlock_deny_access_net, * * @hierarchy: Denying domain's hierarchy node (never NULL); its id is the * domain field. - * @same_exec: Whether the current task entered the denying domain itself. + * @same_exec: Whether the tracer entered the denying domain itself. * @logged: The domain's audit-logging decision for this denial. * @tracee_domain_id: The tracee's Landlock domain ID, or 0 if the tracee * is unsandboxed. * @tracee: The target task ptrace acted on (never NULL). tracee_pid is * the init-namespace TGID (like audit's opid). + * @tracer: The tracer or proposed tracer (never NULL); for PTRACE_TRACEME + * this is the parent, not the syscall caller. * * Emitted when a Landlock domain denies a ptrace operation. */ @@ -837,15 +893,16 @@ TRACE_EVENT(landlock_deny_ptrace, TP_PROTO(const struct landlock_hierarchy *hierarchy, bool same_exec, bool logged, u64 tracee_domain_id, - const struct task_struct *tracee), + const struct task_struct *tracee, + const struct task_struct *tracer), - TP_ARGS(hierarchy, same_exec, logged, tracee_domain_id, tracee), + TP_ARGS(hierarchy, same_exec, logged, tracee_domain_id, tracee, tracer), TP_STRUCT__entry( - __field( __u64, domain_id ) + __field( u64, domain_id ) __field( bool, same_exec ) __field( bool, logged ) - __field( __u64, tracee_domain_id) + __field( u64, tracee_domain_id) __field( pid_t, tracee_pid ) __string( tracee_comm, tracee->comm ) ), @@ -872,12 +929,14 @@ TRACE_EVENT(landlock_deny_ptrace, * * @hierarchy: Denying domain's hierarchy node (never NULL); its id is the * domain field. - * @same_exec: Whether the current task entered the denying domain itself. + * @same_exec: Whether the policy subject entered the denying domain itself. * @logged: The domain's audit-logging decision for this denial. * @target_domain_id: The target's Landlock domain ID, or 0 if the target * is unsandboxed. * @target: The task the signal was aimed at (never NULL). target_pid is * the init-namespace TGID (like audit's opid). + * @signal: The signal selected by the denied check. Zero is a permission + * probe, not an absent value. * * Emitted when a Landlock domain denies signal delivery to a scoped-out * target. @@ -886,15 +945,15 @@ TRACE_EVENT(landlock_deny_scope_signal, TP_PROTO(const struct landlock_hierarchy *hierarchy, bool same_exec, bool logged, u64 target_domain_id, - const struct task_struct *target), + const struct task_struct *target, int signal), - TP_ARGS(hierarchy, same_exec, logged, target_domain_id, target), + TP_ARGS(hierarchy, same_exec, logged, target_domain_id, target, signal), TP_STRUCT__entry( - __field( __u64, domain_id ) + __field( u64, domain_id ) __field( bool, same_exec ) __field( bool, logged ) - __field( __u64, target_domain_id) + __field( u64, target_domain_id) __field( pid_t, target_pid ) __string( target_comm, target->comm ) ), @@ -940,10 +999,10 @@ TRACE_EVENT(landlock_deny_scope_abstract_unix_socket, TP_ARGS(hierarchy, same_exec, logged, peer_domain_id, peer), TP_STRUCT__entry( - __field( __u64, domain_id ) + __field( u64, domain_id ) __field( bool, same_exec ) __field( bool, logged ) - __field( __u64, peer_domain_id ) + __field( u64, peer_domain_id ) __field( pid_t, peer_pid ) /* * Abstract socket names are untrusted binary data from @@ -973,11 +1032,10 @@ TRACE_EVENT(landlock_deny_scope_abstract_unix_socket, __entry->logged = logged; __entry->peer_domain_id = peer_domain_id; /* - * Best-effort (0 for a datagram peer). sk_peer_pid is - * canonically guarded by sk->sk_peer_lock, but the target - * peer's peercred is set once and not updated concurrently in - * these hooks, so this READ_ONCE() is safe; sun_path is the - * reliable identifier. + * Best-effort (0 for a datagram peer). The caller holds the + * peer's AF_UNIX state lock, serializing published peercred + * updates. The peer socket keeps a reference to sk_peer_pid + * through pid_nr(); sun_path is the reliable identifier. */ peer_pid = READ_ONCE(peer->sk_peer_pid); __entry->peer_pid = peer_pid ? pid_nr(peer_pid) : 0; @@ -991,6 +1049,7 @@ TRACE_EVENT(landlock_deny_scope_abstract_unix_socket, __get_dynamic_array_len(sun_path) - 1)) ); +#undef _LANDLOCK_FS_BLOCKER_TYPE_NAMES #undef _LANDLOCK_NAME_ENTRY #endif /* _TRACE_LANDLOCK_H */ diff --git a/include/uapi/linux/alloc_tag.h b/include/uapi/linux/alloc_tag.h index e3ad94444864..7d4618bea043 100644 --- a/include/uapi/linux/alloc_tag.h +++ b/include/uapi/linux/alloc_tag.h @@ -38,6 +38,7 @@ struct allocinfo_counter { __u64 bytes; __u64 calls; __u8 accurate; + __u8 pad[7]; } __attribute__((aligned(8))); struct allocinfo_tag_data { diff --git a/include/uapi/linux/random.h b/include/uapi/linux/random.h index 1dd047ec98a1..ef6614d9c704 100644 --- a/include/uapi/linux/random.h +++ b/include/uapi/linux/random.h @@ -58,7 +58,7 @@ struct rand_pool_info { /** * struct vgetrandom_opaque_params - arguments for allocating memory for vgetrandom * - * @size_per_opaque_state: Size of each state that is to be passed to vgetrandom(). + * @size_of_opaque_state: Size of each state passed to vgetrandom(). * @mmap_prot: Value of the prot argument in mmap(2). * @mmap_flags: Value of the flags argument in mmap(2). * @reserved: Reserved for future use. diff --git a/include/vdso/getrandom.h b/include/vdso/getrandom.h index 6ca4d6de9e46..1d9dbd4d76c1 100644 --- a/include/vdso/getrandom.h +++ b/include/vdso/getrandom.h @@ -64,7 +64,7 @@ extern void __arch_chacha20_blocks_nostack(u8 *dst_bytes, const u32 *key, u32 *c * @opaque_state: Passed to __cvdso_getrandom(). * @opaque_len: Passed to __cvdso_getrandom(); * - * This function is implemented by making a single call to to __cvdso_getrandom(), whose + * This function is implemented by making a single call to __cvdso_getrandom(), whose * documentation may be consulted for more information. * * Returns: The return value of __cvdso_getrandom(). diff --git a/init/Kconfig b/init/Kconfig index 8583d9f06c52..ad592fdf29af 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -173,6 +173,11 @@ config CC_HAS_ALLOC_TOKEN config CC_HAS_MULTIDIMENSIONAL_NONSTRING def_bool $(success,echo 'char tag[][4] __attribute__((__nonstring__)) = { };' | $(CC) $(CLANG_FLAGS) -x c - -c -o /dev/null -Werror) +config CC_OPT_INLINE_MEMSET + string + default "-finline-stringops=memset" if $(cc-option,-finline-stringops=memset) + default "-mllvm -max-store-memset=4294967294" if $(cc-option,-mllvm -max-store-memset=4294967294) + config LD_CAN_USE_KEEP_IN_OVERLAY # ld.lld prior to 21.0.0 did not support KEEP within an overlay description # https://github.com/llvm/llvm-project/pull/130661 diff --git a/io_uring/bpf_filter.c b/io_uring/bpf_filter.c index c0037632b7af..4a21511c4811 100644 --- a/io_uring/bpf_filter.c +++ b/io_uring/bpf_filter.c @@ -253,6 +253,7 @@ void io_bpf_filter_clone(struct io_restriction *dst, struct io_restriction *src) * If the src filter is going away, just ignore it. */ if (refcount_inc_not_zero(&src->bpf_filters->refs)) { + src->bpf_filters_cow = true; dst->bpf_filters = src->bpf_filters; dst->bpf_filters_cow = true; } diff --git a/io_uring/cmd_net.c b/io_uring/cmd_net.c index 7cd411fc4f33..90d4ec7cc761 100644 --- a/io_uring/cmd_net.c +++ b/io_uring/cmd_net.c @@ -69,8 +69,8 @@ static inline int io_uring_cmd_setsockopt(struct socket *sock, optlen); } -static bool io_process_timestamp_skb(struct io_uring_cmd *cmd, struct sock *sk, - struct sk_buff *skb, unsigned issue_flags) +static int io_process_timestamp_skb(struct io_uring_cmd *cmd, struct sock *sk, + struct sk_buff *skb, unsigned int issue_flags) { struct sock_exterr_skb *serr = SKB_EXT_ERR(skb); struct io_uring_cqe cqe[2]; @@ -83,7 +83,7 @@ static bool io_process_timestamp_skb(struct io_uring_cmd *cmd, struct sock *sk, ret = skb_get_tx_timestamp(skb, sk, &ts); if (ret < 0) - return false; + return ret; tskey = serr->ee.ee_data; tstype = serr->ee.ee_info; @@ -98,7 +98,9 @@ static bool io_process_timestamp_skb(struct io_uring_cmd *cmd, struct sock *sk, iots = (struct io_timespec *)&cqe[1]; iots->tv_sec = ts.tv_sec; iots->tv_nsec = ts.tv_nsec; - return io_uring_cmd_post_mshot_cqe32(cmd, issue_flags, cqe); + if (!io_uring_cmd_post_mshot_cqe32(cmd, issue_flags, cqe)) + return -ENOBUFS; + return 0; } static int io_uring_cmd_timestamp(struct socket *sock, @@ -135,7 +137,8 @@ static int io_uring_cmd_timestamp(struct socket *sock, skb = skb_peek(&list); if (!skb) break; - if (!io_process_timestamp_skb(cmd, sk, skb, issue_flags)) + ret = io_process_timestamp_skb(cmd, sk, skb, issue_flags); + if (ret) break; __skb_dequeue(&list); consume_skb(skb); @@ -145,6 +148,12 @@ static int io_uring_cmd_timestamp(struct socket *sock, scoped_guard(spinlock_irqsave, &q->lock) skb_queue_splice(&list, q); } + /* + * Aux CQEs cannot overflow and the poll is edge triggered, so nothing + * re-runs the command once the CQ drains. End the multishot instead. + */ + if (ret == -ENOBUFS) + return -ENOBUFS; return -EAGAIN; } diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 61053421d809..cbea989c5a7c 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -733,7 +733,8 @@ bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32) * Post dummy CQE if a 32b CQE is needed and there's only room for a * 16b CQE before the ring wraps. */ - if (cqe32 && off + 1 == ctx->cq_entries) { + if (cqe32 && (ctx->flags & IORING_SETUP_CQE_MIXED) && + off + 1 == ctx->cq_entries) { if (!io_fill_nop_cqe(ctx, off)) return false; off = 0; @@ -742,12 +743,13 @@ bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32) free = ctx->cq_entries - io_cqring_queued(ctx); /* we need a contiguous range, limit based on the current array offset */ len = min(free, ctx->cq_entries - off); - if (len < (cqe32 + 1)) - return false; - if (ctx->flags & IORING_SETUP_CQE32) { + if (!len) + return false; off <<= 1; len <<= 1; + } else if (len < (cqe32 + 1)) { + return false; } ctx->cqe_cached = &rings->cqes[off]; @@ -781,7 +783,7 @@ static bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res, WRITE_ONCE(cqe->res, res); WRITE_ONCE(cqe->flags, cflags); - if (cqe32) { + if (cqe32 || (ctx->flags & IORING_SETUP_CQE32)) { WRITE_ONCE(cqe->big_cqe[0], 0); WRITE_ONCE(cqe->big_cqe[1], 0); } @@ -2403,8 +2405,8 @@ static __cold void io_ring_exit_work(struct work_struct *work) spin_lock(&ctx->completion_lock); spin_unlock(&ctx->completion_lock); - /* pairs with RCU read section in io_req_local_work_add() */ - if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) + /* pairs with the RCU read sections in the task_work add paths */ + if (ctx->flags & (IORING_SETUP_DEFER_TASKRUN | IORING_SETUP_SQPOLL)) synchronize_rcu(); io_ring_ctx_free(ctx); diff --git a/io_uring/loop.c b/io_uring/loop.c index bbbb6ef14e6a..3d472b7827fa 100644 --- a/io_uring/loop.c +++ b/io_uring/loop.c @@ -2,6 +2,7 @@ #include "io_uring.h" #include "wait.h" #include "loop.h" +#include "tctx.h" static inline int io_loop_nr_cqes(const struct io_ring_ctx *ctx, const struct iou_loop_params *lp) @@ -81,6 +82,10 @@ int io_run_loop(struct io_ring_ctx *ctx) { int ret; + ret = io_uring_add_tctx_node(ctx); + if (unlikely(ret)) + return ret; + if (!io_allowed_run_tw(ctx)) return -EEXIST; diff --git a/io_uring/tw.c b/io_uring/tw.c index f573bcc3af6a..f9affd68dd26 100644 --- a/io_uring/tw.c +++ b/io_uring/tw.c @@ -210,6 +210,9 @@ void io_req_normal_work_add(struct io_kiocb *req) struct io_uring_task *tctx = req->tctx; struct io_ring_ctx *ctx = req->ctx; + /* SQPOLL can retire the request on push, see io_ring_exit_work() */ + guard(rcu)(); + /* tw run already pending, nothing else to do */ if (!mpscq_push(&tctx->task_list, &req->io_task_work.node)) return; diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 86d580d4410d..dcff0b995d34 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -374,6 +374,7 @@ struct io_zcrx_args { struct io_kiocb *req; struct io_zcrx_ifq *ifq; unsigned nr_skbs; + bool stopped_early; }; static const struct memory_provider_ops io_uring_pp_zc_ops; @@ -1899,6 +1900,9 @@ io_zcrx_recv_skb(read_descriptor_t *desc, struct sk_buff *skb, } out: + /* Bytes left in len mean an error stopped the walk early. */ + if (len) + args->stopped_early = true; if (offset == start_off) return ret; desc->count -= (offset - start_off); @@ -1935,8 +1939,9 @@ static int io_zcrx_tcp_recvmsg(struct io_kiocb *req, struct io_zcrx_ifq *ifq, ret = -ENOTCONN; else ret = -EAGAIN; - } else if (unlikely(args.nr_skbs > IO_SKBS_PER_CALL_LIMIT) && - (issue_flags & IO_URING_F_MULTISHOT)) { + } else if ((issue_flags & IO_URING_F_MULTISHOT) && + (unlikely(args.nr_skbs > IO_SKBS_PER_CALL_LIMIT) || + args.stopped_early)) { ret = IOU_REQUEUE; } else if (sock_flag(sk, SOCK_DONE)) { /* Make it to retry until it finally gets 0. */ diff --git a/kernel/audit_tree.c b/kernel/audit_tree.c index 1ed19b775912..f2e81be8265e 100644 --- a/kernel/audit_tree.c +++ b/kernel/audit_tree.c @@ -545,21 +545,37 @@ static void kill_rules(struct audit_context *context, struct audit_tree *tree) { struct audit_krule *rule, *next; struct audit_entry *entry; + bool need_sync = false; + + list_for_each_entry_safe(rule, next, &tree->rules, rlist) { + entry = container_of(rule, struct audit_entry, rule); + + if (rule->tree) { + /* not a half-baked one */ + audit_tree_log_remove_rule(context, rule); + rule->tree = NULL; + list_del_rcu(&entry->list); + list_del(&entry->rule.list); + if (entry->rule.exe) + need_sync = true; + } else { + list_del_init(&rule->rlist); + } + } + + if (list_empty(&tree->rules)) + return; + + if (need_sync) + synchronize_rcu(); list_for_each_entry_safe(rule, next, &tree->rules, rlist) { entry = container_of(rule, struct audit_entry, rule); list_del_init(&rule->rlist); - if (rule->tree) { - /* not a half-baked one */ - audit_tree_log_remove_rule(context, rule); - if (entry->rule.exe) - audit_remove_mark(entry->rule.exe); - rule->tree = NULL; - list_del_rcu(&entry->list); - list_del(&entry->rule.list); - call_rcu(&entry->rcu, audit_free_rule_rcu); - } + if (entry->rule.exe) + audit_remove_mark(entry->rule.exe); + call_rcu(&entry->rcu, audit_free_rule_rcu); } } diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 9f33e95d5741..d870bc5e50bc 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -4268,13 +4268,10 @@ int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec) { int i; - /* There are three types that signify ownership of some other type: - * kptr_ref, bpf_list_head, bpf_rb_root. - * kptr_ref only supports storing kernel types, which can't store - * references to program allocated local types. - * - * Hence we only need to ensure that bpf_{list_head,rb_root} ownership - * does not form cycles. + /* + * Check fields which require the complete BTF and initialize runtime + * metadata. Ownership relationships are validated after every record has + * been fixed up. */ if (IS_ERR_OR_NULL(rec) || !(rec->field_mask & (BPF_GRAPH_ROOT | BPF_UPTR))) return 0; @@ -4305,53 +4302,90 @@ int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec) if (!meta) return -EFAULT; rec->fields[i].graph_root.value_rec = meta->record; - - /* We need to set value_rec for all root types, but no need - * to check ownership cycle for a type unless it's also a - * node type. - */ - if (!(rec->field_mask & BPF_GRAPH_NODE)) - continue; - - /* We need to ensure ownership acyclicity among all types. The - * proper way to do it would be to topologically sort all BTF - * IDs based on the ownership edges, since there can be multiple - * bpf_{list_head,rb_node} in a type. Instead, we use the - * following resaoning: - * - * - A type can only be owned by another type in user BTF if it - * has a bpf_{list,rb}_node. Let's call these node types. - * - A type can only _own_ another type in user BTF if it has a - * bpf_{list_head,rb_root}. Let's call these root types. - * - * We ensure that if a type is both a root and node, its - * element types cannot be root types. - * - * To ensure acyclicity: - * - * When A is an root type but not a node, its ownership - * chain can be: - * A -> B -> C - * Where: - * - A is an root, e.g. has bpf_rb_root. - * - B is both a root and node, e.g. has bpf_rb_node and - * bpf_list_head. - * - C is only an root, e.g. has bpf_list_node - * - * When A is both a root and node, some other type already - * owns it in the BTF domain, hence it can not own - * another root type through any of the ownership edges. - * A -> B - * Where: - * - A is both an root and node. - * - B is only an node. - */ - if (meta->record->field_mask & BPF_GRAPH_ROOT) - return -ELOOP; } return 0; } +static int btf_owned_type_idx(const struct btf *btf, struct btf_struct_metas *tab, + const struct btf_field *field) +{ + struct btf_struct_meta *meta; + u32 btf_id; + + if (field->type & BPF_GRAPH_ROOT) { + btf_id = field->graph_root.value_btf_id; + } else if (field->type == BPF_KPTR_REF || field->type == BPF_KPTR_PERCPU) { + if (btf_is_kernel(field->kptr.btf)) + return -ENOENT; + btf_id = field->kptr.btf_id; + } else { + return -ENOENT; + } + + meta = btf_find_struct_meta(btf, btf_id); + if (!meta) + return field->type & BPF_GRAPH_ROOT ? -EFAULT : -ENOENT; + return meta - tab->types; +} + +/* + * Each ownership edge adds kernel frames through bpf_obj_free_fields() and + * __bpf_obj_drop_impl(). Keep the bound deliberately small because object + * destruction can itself run below a BPF call chain. A final pointee without + * special fields is not present in the struct metadata table and adds only a + * non-recursing drop. + */ +#define BTF_MAX_OWNERSHIP_DEPTH 8 + +static int btf_ownership_depth(const struct btf *btf, + struct btf_struct_metas *tab, u8 *depth, + int idx, int depth_left) +{ + const struct btf_record *rec = tab->types[idx].record; + int i, ret, max_depth = 0; + + if (!depth_left) + return -ELOOP; + if (depth[idx]) + goto done; + + for (i = 0; i < rec->cnt; i++) { + ret = btf_owned_type_idx(btf, tab, &rec->fields[i]); + if (ret == -ENOENT) + continue; + if (ret < 0) + return ret; + ret = btf_ownership_depth(btf, tab, depth, ret, depth_left - 1); + if (ret < 0) + return ret; + max_depth = max(max_depth, ret); + } + depth[idx] = max_depth + 1; +done: + return depth[idx] > depth_left ? -ELOOP : depth[idx]; +} + +static int btf_check_ownership_depth(const struct btf *btf, + struct btf_struct_metas *tab) +{ + u8 *depth; + int i, ret = 0; + + depth = kvcalloc(tab->cnt, sizeof(*depth), GFP_KERNEL | __GFP_NOWARN); + if (!depth) + return -ENOMEM; + + for (i = 0; i < tab->cnt; i++) { + ret = btf_ownership_depth(btf, tab, depth, i, + BTF_MAX_OWNERSHIP_DEPTH); + if (ret < 0) + break; + ret = 0; + } + kvfree(depth); + return ret; +} + static void __btf_struct_show(const struct btf *btf, const struct btf_type *t, u32 type_id, void *data, u8 bits_offset, struct btf_show *show) @@ -6044,6 +6078,10 @@ static struct btf *btf_parse(const union bpf_attr *attr, bpfptr_t uattr, if (err < 0) goto errout_meta; } + + err = btf_check_ownership_depth(btf, struct_meta_tab); + if (err < 0) + goto errout_meta; } err = bpf_log_attr_finalize(attr_log, &env->log); @@ -7187,7 +7225,7 @@ static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, if (btf_type_is_int(t)) return WALK_SCALAR; - if (!btf_type_is_struct(t)) + if (!btf_type_is_struct(t) || !t->size) goto error; off = (off - moff) % t->size; diff --git a/kernel/bpf/check_btf.c b/kernel/bpf/check_btf.c index 0e8b3ccc7a5b..4c1ed842f661 100644 --- a/kernel/bpf/check_btf.c +++ b/kernel/bpf/check_btf.c @@ -338,9 +338,9 @@ static int check_btf_line(struct bpf_verifier_env *env, #define MIN_CORE_RELO_SIZE sizeof(struct bpf_core_relo) #define MAX_CORE_RELO_SIZE MAX_FUNCINFO_REC_SIZE -static int check_core_relo(struct bpf_verifier_env *env, - const union bpf_attr *attr, - bpfptr_t uattr) +int bpf_check_core_relo(struct bpf_verifier_env *env, + const union bpf_attr *attr, + bpfptr_t uattr) { u32 i, nr_core_relo, ncopy, expected_size, rec_size; struct bpf_core_relo core_relo = {}; @@ -414,7 +414,7 @@ int bpf_prepare_btf_info(struct bpf_verifier_env *env, struct btf *btf; int err; - if (!attr->func_info_cnt && !attr->line_info_cnt) { + if (!attr->func_info_cnt && !attr->line_info_cnt && !attr->core_relo_cnt) { if (check_abnormal_return(env)) return -EINVAL; return 0; @@ -455,9 +455,5 @@ int bpf_check_btf_info(struct bpf_verifier_env *env, if (err) return err; - err = check_core_relo(env, attr, uattr); - if (err) - return err; - return 0; } diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 8b294dfc1ad4..7f11555a5070 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -1361,7 +1362,7 @@ static int bpf_jit_blind_insn(const struct bpf_insn *from, { struct bpf_insn *to = to_buff; u32 imm_rnd = get_random_u32(); - s16 off; + int off; BUILD_BUG_ON(BPF_REG_PARAMS + 2 != MAX_BPF_JIT_REG); BUILD_BUG_ON(BPF_REG_AX + 1 != MAX_BPF_JIT_REG); @@ -1437,6 +1438,8 @@ static int bpf_jit_blind_insn(const struct bpf_insn *from, off = from->off; if (off < 0) off -= 2; + if (off < S16_MIN) + return -ERANGE; *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); *to++ = BPF_JMP_REG(from->code, from->dst_reg, BPF_REG_AX, off); @@ -1457,6 +1460,8 @@ static int bpf_jit_blind_insn(const struct bpf_insn *from, off = from->off; if (off < 0) off -= 2; + if (off < S16_MIN) + return -ERANGE; *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); *to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); *to++ = BPF_JMP32_REG(from->code, from->dst_reg, BPF_REG_AX, @@ -1605,7 +1610,9 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bp if (!rewritten) continue; - if (env) + if (rewritten < 0) + tmp = ERR_PTR(rewritten); + else if (env) tmp = bpf_patch_insn_data(env, i, insn_buff, rewritten); else tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten); @@ -1619,6 +1626,8 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bp * fix it up here on error. */ bpf_jit_prog_release_other(prog, clone); + if (env && fatal_signal_pending(current)) + return ERR_PTR(-EINTR); return IS_ERR(tmp) ? tmp : ERR_PTR(-ENOMEM); } @@ -2636,11 +2645,14 @@ static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struc orig_prog = prog; prog = bpf_jit_blind_constants(env, prog); /* - * If blinding was requested and we failed during blinding, we must fall - * back to the interpreter. + * Fall back to the interpreter after blinding failures, except when + * the loader was killed. */ - if (IS_ERR(prog)) + if (IS_ERR(prog)) { + if (PTR_ERR(prog) == -EINTR) + return prog; goto out_restore; + } prog = bpf_int_jit_compile(env, prog); if (prog->jited) { @@ -2659,6 +2671,8 @@ static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struc struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct bpf_prog *fp, int *err) { + struct bpf_prog *jit_prog; + /* In case of BPF to BPF calls, verifier did all the prep * work with regards to JITing, etc. */ @@ -2681,7 +2695,12 @@ struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct if (*err) return fp; - fp = bpf_prog_jit_compile(env, fp); + jit_prog = bpf_prog_jit_compile(env, fp); + if (IS_ERR(jit_prog)) { + *err = PTR_ERR(jit_prog); + return fp; + } + fp = jit_prog; bpf_prog_jit_attempt_done(fp); if (!fp->jited && jit_needed) { *err = -ENOTSUPP; diff --git a/kernel/bpf/crypto.c b/kernel/bpf/crypto.c index 51f89cecefb4..3f3fe2450fc6 100644 --- a/kernel/bpf/crypto.c +++ b/kernel/bpf/crypto.c @@ -149,8 +149,9 @@ bpf_crypto_ctx_create(const struct bpf_crypto_params *params, u32 params__sz, const struct bpf_crypto_type *type; struct bpf_crypto_ctx *ctx; - if (!params || params->reserved[0] || params->reserved[1] || - params__sz != sizeof(struct bpf_crypto_params)) { + if (!params || + params__sz != sizeof(struct bpf_crypto_params) || + params->reserved[0] || params->reserved[1]) { *err = -EINVAL; return NULL; } diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 52d3cec33672..d6f83521fc78 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include "disasm.h" @@ -306,12 +307,28 @@ static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) } } +/* + * Some post-verification instruction rewriting passes require an + * O(prog->len) operation per instruction. Keep their shared primitives + * killable and preemptible. + */ +static bool bpf_rewrite_must_abort(void) +{ + if (fatal_signal_pending(current)) + return true; + cond_resched(); + return false; +} + struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, const struct bpf_insn *patch, u32 len) { struct bpf_prog *new_prog; struct bpf_insn_aux_data *new_data = NULL; + if (bpf_rewrite_must_abort()) + return NULL; + if (len > 1) { new_data = vrealloc(env->insn_aux_data, array_size(env->prog->len + len - 1, @@ -523,6 +540,9 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) unsigned int orig_prog_len = env->prog->len; int err; + if (bpf_rewrite_must_abort()) + return -EINTR; + if (bpf_prog_is_offloaded(env->prog->aux)) bpf_prog_offload_remove_insns(env, off, cnt); @@ -1356,7 +1376,7 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) } prog = bpf_jit_blind_constants(env, prog); if (IS_ERR(prog)) { - err = -ENOMEM; + err = PTR_ERR(prog); prog = orig_prog; goto out_restore; } @@ -1433,7 +1453,7 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env) err = bpf_jit_subprogs(env); if (err == 0) return 0; - if (err == -EFAULT) + if (err == -EFAULT || err == -EINTR) return err; } #ifndef CONFIG_BPF_JIT_ALWAYS_ON diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 6f331c80130d..13a2356c84cf 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -128,6 +128,7 @@ struct htab_elem { struct htab_btf_record { struct btf_record *record; + struct btf *btf; u32 key_size; }; @@ -497,8 +498,13 @@ static void htab_dtor_ctx_free(void *ctx) { struct htab_btf_record *hrec = ctx; + /* + * The duplicated record still points into the map BTF, so free it + * before dropping the reference that keeps that BTF alive. + */ btf_record_free(hrec->record); - kfree(ctx); + btf_put(hrec->btf); + kfree(hrec); } static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma, @@ -521,6 +527,15 @@ static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma, kfree(hrec); return err; } + /* + * btf_record_dup() only acquires kernel and module BTF. Fields whose + * types live in the map BTF keep pointing into it: kptrs to local + * types refer to map->btf, and graph roots carry a value record owned + * by its struct meta table. The context can outlive the map when the + * allocator defers its teardown, so hold a reference of our own. + */ + hrec->btf = map->btf; + btf_get(hrec->btf); bpf_mem_alloc_set_dtor(ma, dtor, htab_dtor_ctx_free, hrec); return 0; } @@ -1054,14 +1069,17 @@ static void pcpu_init_value(struct bpf_htab *htab, void __percpu *pptr, /* When not setting the initial value on all cpus, zero-fill element * values for other cpus. Otherwise, bpf program has no way to ensure * known initial values for cpus other than current one - * (onallcpus=false always when coming from bpf prog). + * (onallcpus=false always when coming from bpf prog, + * map_flags & BPF_F_CPU when coming from syscall but setting + * only one cpu). */ - if (!onallcpus) { - int current_cpu = raw_smp_processor_id(); + if (!onallcpus || (map_flags & BPF_F_CPU)) { + int init_cpu = (map_flags & BPF_F_CPU) ? map_flags >> 32 : + raw_smp_processor_id(); int cpu; for_each_possible_cpu(cpu) { - if (cpu == current_cpu) + if (cpu == init_cpu) copy_map_value(&htab->map, per_cpu_ptr(pptr, cpu), value); else /* Since elem is preallocated, we cannot touch special fields */ zero_map_value(&htab->map, per_cpu_ptr(pptr, cpu)); @@ -1772,6 +1790,12 @@ static int htab_lru_percpu_map_lookup_and_delete_elem(struct bpf_map *map, flags); } +/* + * Max consecutive empty buckets to walk in one RCU + + * instrumentation-disabled section before rescheduling. + */ +#define HTAB_BATCH_EMPTY_RESCHED 64 + static int __htab_map_lookup_and_delete_batch(struct bpf_map *map, const union bpf_attr *attr, @@ -1793,6 +1817,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map, unsigned long flags = 0; bool locked = false; struct htab_elem *l; + u32 empty_cnt = 0; struct bucket *b; int ret = 0; @@ -1971,30 +1996,41 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map, } next_batch: - /* If we are not copying data, we can go to next bucket and avoid - * unlocking the rcu. + /* + * If we are not copying data, we can go to next bucket and avoid + * unlocking the rcu. Bound the walk though: after + * HTAB_BATCH_EMPTY_RESCHED consecutive empty buckets, fully exit + * the critical section (no locks are held here) and reschedule. */ if (!bucket_cnt && (batch + 1 < htab->n_buckets)) { batch++; - goto again_nocopy; + if (++empty_cnt < HTAB_BATCH_EMPTY_RESCHED) + goto again_nocopy; + empty_cnt = 0; + rcu_read_unlock(); + bpf_enable_instrumentation(); + cond_resched_tasks_rcu_qs(); + goto again; } rcu_read_unlock(); bpf_enable_instrumentation(); - if (bucket_cnt && (copy_to_user(ukeys + total * key_size, keys, - key_size * bucket_cnt) || - copy_to_user(uvalues + total * value_size, values, - value_size * bucket_cnt))) { + if (bucket_cnt && (copy_to_user(ukeys + (size_t)total * key_size, keys, + (size_t)key_size * bucket_cnt) || + copy_to_user(uvalues + (size_t)total * value_size, values, + (size_t)value_size * bucket_cnt))) { ret = -EFAULT; goto after_loop; } total += bucket_cnt; + empty_cnt = 0; batch++; if (batch >= htab->n_buckets) { ret = -ENOENT; goto after_loop; } + cond_resched_tasks_rcu_qs(); goto again; after_loop: @@ -3338,8 +3374,10 @@ static int __rhtab_map_lookup_and_delete_batch(struct bpf_map *map, } if (do_delete) { + migrate_disable(); for (i = 0; i < total; i++) rhtab_delete_elem(rhtab, del_elems[i], NULL, 0); + migrate_enable(); } rcu_read_unlock(); diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index b3cc5c8fc875..712dca5a2c5b 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -4883,7 +4883,7 @@ BTF_ID(func, bpf_cgroup_release_dtor) BTF_KFUNCS_START(common_btf_ids) BTF_ID_FLAGS(func, bpf_cast_to_kern_ctx, KF_FASTCALL) -BTF_ID_FLAGS(func, bpf_rdonly_cast, KF_FASTCALL) +BTF_ID_FLAGS(func, bpf_rdonly_cast, KF_FASTCALL | KF_PERFMON) BTF_ID_FLAGS(func, bpf_rcu_read_lock) BTF_ID_FLAGS(func, bpf_rcu_read_unlock) BTF_ID_FLAGS(func, bpf_dynptr_slice, KF_RET_NULL) @@ -4920,26 +4920,26 @@ BTF_ID_FLAGS(func, bpf_wq_set_callback, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_wq_start) BTF_ID_FLAGS(func, bpf_preempt_disable) BTF_ID_FLAGS(func, bpf_preempt_enable) -BTF_ID_FLAGS(func, bpf_iter_bits_new, KF_ITER_NEW) +BTF_ID_FLAGS(func, bpf_iter_bits_new, KF_ITER_NEW | KF_PERFMON) BTF_ID_FLAGS(func, bpf_iter_bits_next, KF_ITER_NEXT | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_iter_bits_destroy, KF_ITER_DESTROY) -BTF_ID_FLAGS(func, bpf_copy_from_user_str, KF_SLEEPABLE) -BTF_ID_FLAGS(func, bpf_copy_from_user_task_str, KF_SLEEPABLE) -BTF_ID_FLAGS(func, bpf_get_kmem_cache) +BTF_ID_FLAGS(func, bpf_copy_from_user_str, KF_SLEEPABLE | KF_PERFMON) +BTF_ID_FLAGS(func, bpf_copy_from_user_task_str, KF_SLEEPABLE | KF_PERFMON) +BTF_ID_FLAGS(func, bpf_get_kmem_cache, KF_PERFMON) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_new, KF_ITER_NEW | KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_iter_kmem_cache_destroy, KF_ITER_DESTROY | KF_SLEEPABLE) BTF_ID_FLAGS(func, bpf_local_irq_save) BTF_ID_FLAGS(func, bpf_local_irq_restore) #ifdef CONFIG_BPF_EVENTS -BTF_ID_FLAGS(func, bpf_probe_read_user_dynptr) -BTF_ID_FLAGS(func, bpf_probe_read_kernel_dynptr) -BTF_ID_FLAGS(func, bpf_probe_read_user_str_dynptr) -BTF_ID_FLAGS(func, bpf_probe_read_kernel_str_dynptr) -BTF_ID_FLAGS(func, bpf_copy_from_user_dynptr, KF_SLEEPABLE) -BTF_ID_FLAGS(func, bpf_copy_from_user_str_dynptr, KF_SLEEPABLE) -BTF_ID_FLAGS(func, bpf_copy_from_user_task_dynptr, KF_SLEEPABLE) -BTF_ID_FLAGS(func, bpf_copy_from_user_task_str_dynptr, KF_SLEEPABLE) +BTF_ID_FLAGS(func, bpf_probe_read_user_dynptr, KF_PERFMON) +BTF_ID_FLAGS(func, bpf_probe_read_kernel_dynptr, KF_PERFMON) +BTF_ID_FLAGS(func, bpf_probe_read_user_str_dynptr, KF_PERFMON) +BTF_ID_FLAGS(func, bpf_probe_read_kernel_str_dynptr, KF_PERFMON) +BTF_ID_FLAGS(func, bpf_copy_from_user_dynptr, KF_SLEEPABLE | KF_PERFMON) +BTF_ID_FLAGS(func, bpf_copy_from_user_str_dynptr, KF_SLEEPABLE | KF_PERFMON) +BTF_ID_FLAGS(func, bpf_copy_from_user_task_dynptr, KF_SLEEPABLE | KF_PERFMON) +BTF_ID_FLAGS(func, bpf_copy_from_user_task_str_dynptr, KF_SLEEPABLE | KF_PERFMON) #endif #ifdef CONFIG_DMA_SHARED_BUFFER BTF_ID_FLAGS(func, bpf_iter_dmabuf_new, KF_ITER_NEW | KF_SLEEPABLE) @@ -4947,26 +4947,26 @@ BTF_ID_FLAGS(func, bpf_iter_dmabuf_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLEEPAB BTF_ID_FLAGS(func, bpf_iter_dmabuf_destroy, KF_ITER_DESTROY | KF_SLEEPABLE) #endif BTF_ID_FLAGS(func, __bpf_trap) -BTF_ID_FLAGS(func, bpf_strcmp); -BTF_ID_FLAGS(func, bpf_strcasecmp); -BTF_ID_FLAGS(func, bpf_strncasecmp); -BTF_ID_FLAGS(func, bpf_strchr); -BTF_ID_FLAGS(func, bpf_strchrnul); -BTF_ID_FLAGS(func, bpf_strnchr); -BTF_ID_FLAGS(func, bpf_strrchr); -BTF_ID_FLAGS(func, bpf_strlen); -BTF_ID_FLAGS(func, bpf_strnlen); -BTF_ID_FLAGS(func, bpf_strspn); -BTF_ID_FLAGS(func, bpf_strcspn); -BTF_ID_FLAGS(func, bpf_strstr); -BTF_ID_FLAGS(func, bpf_strcasestr); -BTF_ID_FLAGS(func, bpf_strnstr); -BTF_ID_FLAGS(func, bpf_strncasestr); +BTF_ID_FLAGS(func, bpf_strcmp, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strcasecmp, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strncasecmp, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strchr, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strchrnul, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strnchr, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strrchr, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strlen, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strnlen, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strspn, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strcspn, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strstr, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strcasestr, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strnstr, KF_PERFMON); +BTF_ID_FLAGS(func, bpf_strncasestr, KF_PERFMON); #if defined(CONFIG_BPF_LSM) && defined(CONFIG_CGROUPS) BTF_ID_FLAGS(func, bpf_cgroup_read_xattr, KF_RCU) #endif -BTF_ID_FLAGS(func, bpf_stream_vprintk, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) -BTF_ID_FLAGS(func, bpf_stream_print_stack, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_stream_vprintk, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE | KF_PERFMON) +BTF_ID_FLAGS(func, bpf_stream_print_stack, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE | KF_PERFMON) BTF_ID_FLAGS(func, bpf_task_work_schedule_signal, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_task_work_schedule_resume, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_dynptr_from_file) diff --git a/kernel/bpf/memalloc.c b/kernel/bpf/memalloc.c index e9662db7198f..15684d0fc883 100644 --- a/kernel/bpf/memalloc.c +++ b/kernel/bpf/memalloc.c @@ -118,7 +118,13 @@ struct bpf_mem_cache { struct llist_head free_by_rcu_ttrace; struct llist_head waiting_for_gp_ttrace; struct rcu_head rcu_ttrace; + /* + * 0 - idle + * 1 - __free_rcu() is queued + * 2 - __free_rcu() is queued and free_by_rcu_ttrace got more objects since + */ atomic_t call_rcu_ttrace_in_progress; + raw_spinlock_t lock; }; struct bpf_mem_caches { @@ -214,25 +220,24 @@ static void alloc_bulk(struct bpf_mem_cache *c, int cnt, int node, bool atomic) gfp = __GFP_NOWARN | __GFP_ACCOUNT; gfp |= atomic ? GFP_NOWAIT : GFP_KERNEL; - for (i = 0; i < cnt; i++) { - /* - * For every 'c' llist_del_first(&c->free_by_rcu_ttrace); is - * done only by one CPU == current CPU. Other CPUs might - * llist_add() and llist_del_all() in parallel. - */ - obj = llist_del_first(&c->free_by_rcu_ttrace); - if (!obj) - break; - add_obj_to_free_list(c, obj); - } - if (i >= cnt) - return; + /* + * c->lock serializes concurrent llist_del_first() against + * llist_del_all() in __free_rcu() and do_call_rcu_ttrace(). + */ + scoped_guard(raw_spinlock_irqsave, &c->lock) { + for (i = 0; i < cnt; i++) { + obj = llist_del_first(&c->free_by_rcu_ttrace); + if (!obj) + break; + add_obj_to_free_list(c, obj); + } - for (; i < cnt; i++) { - obj = llist_del_first(&c->waiting_for_gp_ttrace); - if (!obj) - break; - add_obj_to_free_list(c, obj); + for (; i < cnt; i++) { + obj = llist_del_first(&c->waiting_for_gp_ttrace); + if (!obj) + break; + add_obj_to_free_list(c, obj); + } } if (i >= cnt) return; @@ -276,12 +281,30 @@ static int free_all(struct bpf_mem_cache *c, struct llist_node *llnode, bool per return cnt; } +static void __do_call_rcu_ttrace(struct bpf_mem_cache *c); + static void __free_rcu(struct rcu_head *head) { struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu_ttrace); + struct llist_node *llnode; - free_all(c, llist_del_all(&c->waiting_for_gp_ttrace), !!c->percpu_size); - atomic_set(&c->call_rcu_ttrace_in_progress, 0); + scoped_guard(raw_spinlock_irqsave, &c->lock) + llnode = llist_del_all(&c->waiting_for_gp_ttrace); + + free_all(c, llnode, !!c->percpu_size); + + /* + * do_call_rcu_ttrace() that ran while GP was in flight left its objects + * in free_by_rcu_ttrace. This cache may never free or alloc in bulk + * again, so start the next GP from here. + * 'c' can be freed as soon as call_rcu_ttrace_in_progress is zero. + */ + if (atomic_cmpxchg(&c->call_rcu_ttrace_in_progress, 1, 0) == 1) + return; + + /* Pairs with synchronize_rcu() in free_mem_alloc() */ + guard(rcu)(); + __do_call_rcu_ttrace(c); } static void enque_to_free(struct bpf_mem_cache *c, void *obj) @@ -294,17 +317,15 @@ static void enque_to_free(struct bpf_mem_cache *c, void *obj) llist_add(llnode, &c->free_by_rcu_ttrace); } -static void do_call_rcu_ttrace(struct bpf_mem_cache *c) +static void __do_call_rcu_ttrace(struct bpf_mem_cache *c) { struct llist_node *llnode, *t; - if (atomic_xchg(&c->call_rcu_ttrace_in_progress, 1)) { - if (unlikely(READ_ONCE(c->draining))) { - llnode = llist_del_all(&c->free_by_rcu_ttrace); - free_all(c, llnode, !!c->percpu_size); - } - return; - } + /* + * Must be done before llist_del_all(). Objects that it misses were + * added by do_call_rcu_ttrace() that will set 2 after this store. + */ + atomic_set(&c->call_rcu_ttrace_in_progress, 1); WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp_ttrace)); llist_for_each_safe(llnode, t, llist_del_all(&c->free_by_rcu_ttrace)) @@ -323,6 +344,22 @@ static void do_call_rcu_ttrace(struct bpf_mem_cache *c) call_rcu_tasks_trace(&c->rcu_ttrace, __free_rcu); } +static void do_call_rcu_ttrace(struct bpf_mem_cache *c) +{ + struct llist_node *llnode; + + if (atomic_xchg(&c->call_rcu_ttrace_in_progress, 2)) { + if (unlikely(READ_ONCE(c->draining))) { + scoped_guard(raw_spinlock_irqsave, &c->lock) + llnode = llist_del_all(&c->free_by_rcu_ttrace); + free_all(c, llnode, !!c->percpu_size); + } + return; + } + + __do_call_rcu_ttrace(c); +} + static void free_bulk(struct bpf_mem_cache *c) { struct bpf_mem_cache *tgt = c->tgt; @@ -535,6 +572,7 @@ int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu) c->objcg = objcg; c->percpu_size = percpu_size; c->tgt = c; + raw_spin_lock_init(&c->lock); init_refill_work(c); prefill_mem_cache(c, cpu); } @@ -557,7 +595,7 @@ int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu) c->objcg = objcg; c->percpu_size = percpu_size; c->tgt = c; - + raw_spin_lock_init(&c->lock); init_refill_work(c); prefill_mem_cache(c, cpu); } @@ -609,7 +647,7 @@ int bpf_mem_alloc_percpu_unit_init(struct bpf_mem_alloc *ma, int size) c->objcg = objcg; c->percpu_size = percpu_size; c->tgt = c; - + raw_spin_lock_init(&c->lock); init_refill_work(c); prefill_mem_cache(c, cpu); } @@ -694,7 +732,12 @@ static void free_mem_alloc(struct bpf_mem_alloc *ma) * to wait for the pending __free_by_rcu(), and __free_rcu(). RCU Tasks * Trace grace period implies RCU grace period, so all __free_rcu don't * need extra call_rcu() (and thus extra rcu_barrier() here). + * + * __free_rcu() queues itself again unless it sees 'draining'. After + * synchronize_rcu() it either did that already or will not do it, so + * rcu_barrier_tasks_trace() cannot miss it. */ + synchronize_rcu(); rcu_barrier(); /* wait for __free_by_rcu */ rcu_barrier_tasks_trace(); /* wait for __free_rcu */ free_mem_alloc_no_barrier(ma); diff --git a/kernel/bpf/offload.c b/kernel/bpf/offload.c index 0d6f5569588c..d855399812ee 100644 --- a/kernel/bpf/offload.c +++ b/kernel/bpf/offload.c @@ -698,6 +698,8 @@ static bool __bpf_offload_dev_match(struct bpf_prog *prog, return false; if (offload->netdev == netdev) return true; + if (!bpf_prog_is_offloaded(prog->aux)) + return false; ondev1 = bpf_offload_find_netdev(offload->netdev); ondev2 = bpf_offload_find_netdev(netdev); diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 66fb11b6c6a7..012b82513a3b 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -491,7 +491,8 @@ static bool regs_exact(const struct bpf_reg_state *rold, { return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 && check_ids(rold->id, rcur->id, idmap) && - check_ids(rold->parent_id, rcur->parent_id, idmap); + check_ids(rold->parent_id, rcur->parent_id, idmap) && + check_ids(rold->map_uid, rcur->map_uid, idmap); } enum exact_level { @@ -616,7 +617,8 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, range_within(rold, rcur) && tnum_in(rold->var_off, rcur->var_off) && check_ids(rold->id, rcur->id, idmap) && - check_ids(rold->parent_id, rcur->parent_id, idmap); + check_ids(rold->parent_id, rcur->parent_id, idmap) && + check_ids(rold->map_uid, rcur->map_uid, idmap); case PTR_TO_PACKET_META: case PTR_TO_PACKET: /* We must have at least as much range as the old ptr @@ -635,14 +637,14 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, /* id relations must be preserved */ if (!check_ids(rold->id, rcur->id, idmap)) return false; + /* Preserve displacements between pointers sharing an ID. */ + if (rold->id && rold->r64.base != rcur->r64.base) + return false; /* new val must satisfy old val knowledge */ return range_within(rold, rcur) && tnum_in(rold->var_off, rcur->var_off); case PTR_TO_STACK: - /* two stack pointers are equal only if they're pointing to - * the same stack frame, since fp-8 in foo != fp-8 in bar - */ - return regs_exact(rold, rcur, idmap) && rold->frameno == rcur->frameno; + return regs_exact(rold, rcur, idmap); case PTR_TO_ARENA: return true; case PTR_TO_INSN: @@ -1121,7 +1123,7 @@ static bool states_maybe_looping(struct bpf_verifier_state *old, fcur = cur->frame[fr]; for (i = 0; i < MAX_BPF_REG; i++) if (memcmp(&fold->regs[i], &fcur->regs[i], - offsetof(struct bpf_reg_state, frameno))) + offsetof(struct bpf_reg_state, precise))) return false; return true; } diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index c7bc9ba9b331..96217b99399d 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2036,7 +2036,7 @@ int generic_map_delete_batch(struct bpf_map *map, for (cp = 0; cp < max_count; cp++) { err = -EFAULT; - if (copy_from_user(key, keys + cp * map->key_size, + if (copy_from_user(key, keys + (size_t)cp * map->key_size, map->key_size)) break; @@ -2098,9 +2098,9 @@ int generic_map_update_batch(struct bpf_map *map, struct file *map_file, for (cp = 0; cp < max_count; cp++) { err = -EFAULT; - if (copy_from_user(key, keys + cp * map->key_size, + if (copy_from_user(key, keys + (size_t)cp * map->key_size, map->key_size) || - copy_from_user(value, values + cp * value_size, value_size)) + copy_from_user(value, values + (size_t)cp * value_size, value_size)) break; err = bpf_map_update_value(map, map_file, key, value, @@ -2179,12 +2179,12 @@ int generic_map_lookup_batch(struct bpf_map *map, if (err) goto free_buf; - if (copy_to_user(keys + cp * map->key_size, key, + if (copy_to_user(keys + (size_t)cp * map->key_size, key, map->key_size)) { err = -EFAULT; goto free_buf; } - if (copy_to_user(values + cp * value_size, value, value_size)) { + if (copy_to_user(values + (size_t)cp * value_size, value, value_size)) { err = -EFAULT; goto free_buf; } @@ -2448,6 +2448,21 @@ static void __bpf_prog_put_rcu(struct rcu_head *rcu) bpf_prog_free(aux->prog); } +/* + * Progs called from a trampoline can also be reached by a task that was + * preempted in the trampoline before the prog's enter helper took its RCU + * read lock, wait for those first. + */ +static void __bpf_prog_put_rcu_tasks(struct rcu_head *rcu) +{ + struct bpf_prog *prog = container_of(rcu, struct bpf_prog_aux, rcu)->prog; + + if (prog->sleepable) + call_rcu_tasks_trace(rcu, __bpf_prog_put_rcu); + else + call_rcu(rcu, __bpf_prog_put_rcu); +} + static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) { bpf_prog_kallsyms_del_all(prog); @@ -2461,7 +2476,9 @@ static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) btf_put(prog->aux->attach_btf); if (deferred) { - if (prog->sleepable) + if (IS_ENABLED(CONFIG_TASKS_RCU) && prog->aux->tramp_linked) + call_rcu_tasks(&prog->aux->rcu, __bpf_prog_put_rcu_tasks); + else if (prog->sleepable) call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu); else call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); @@ -6042,7 +6059,10 @@ struct bpf_link *bpf_link_get_curr_or_next(u32 *id) again: link = idr_get_next(&link_idr, id); if (link) { - link = bpf_link_inc_not_zero(link); + if (link->id) + link = bpf_link_inc_not_zero(link); + else + link = ERR_PTR(-EAGAIN); if (IS_ERR(link)) { (*id)++; goto again; diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 90b70ea0d370..bf4cb3dd444d 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -401,6 +401,7 @@ static struct bpf_trampoline *bpf_trampoline_lookup(u64 key, unsigned long ip) head = &trampoline_ip_table[hash_64(tr->ip, TRAMPOLINE_HASH_BITS)]; hlist_add_head(&tr->hlist_ip, head); refcount_set(&tr->refcnt, 1); + INIT_LIST_HEAD(&tr->images); for (i = 0; i < BPF_TRAMP_MAX; i++) INIT_HLIST_HEAD(&tr->progs_hlist[i]); out: @@ -565,15 +566,22 @@ static void bpf_tramp_image_free(struct bpf_tramp_image *im) arch_free_bpf_trampoline(im->image, im->size); bpf_jit_uncharge_modmem(im->size); percpu_ref_exit(&im->pcref); + kfree(im->skips); kfree_rcu(im, rcu); } static void __bpf_tramp_image_put_deferred(struct work_struct *work) { struct bpf_tramp_image *im; + struct bpf_trampoline *tr; im = container_of(work, struct bpf_tramp_image, work); + tr = im->tr; + trampoline_lock(tr); + list_del(&im->list); + trampoline_unlock(tr); bpf_tramp_image_free(im); + bpf_trampoline_put(tr); } /* callback, fexit step 3 or fentry step 2 */ @@ -601,7 +609,7 @@ static void __bpf_tramp_image_put_rcu_tasks(struct rcu_head *rcu) struct bpf_tramp_image *im; im = container_of(rcu, struct bpf_tramp_image, rcu); - if (im->ip_after_call) + if (im->call_orig) /* the case of fmod_ret/fexit trampoline and CONFIG_PREEMPTION=y */ percpu_ref_kill(&im->pcref); else @@ -621,9 +629,9 @@ static void bpf_tramp_image_put(struct bpf_tramp_image *im) * * The trampoline is unreachable before bpf_tramp_image_put(). * - * First, patch the trampoline to avoid calling into fexit progs. - * The progs will be freed even if the original function is still - * executing or sleeping. + * Progs are patched out of the image when they are detached, see + * bpf_trampoline_skip_prog(), so they can be freed even if a task is + * still in the image. * In case of CONFIG_PREEMPT=y use call_rcu_tasks() to wait on * first few asm instructions to execute and call into * __bpf_tramp_enter->percpu_ref_get. @@ -637,11 +645,7 @@ static void bpf_tramp_image_put(struct bpf_tramp_image *im) * percpu_ref_kill will be waiting for. Hence the first * call_rcu_tasks() is not necessary. */ - if (im->ip_after_call) { - int err = bpf_arch_text_poke(im->ip_after_call, BPF_MOD_NOP, - BPF_MOD_JUMP, NULL, - im->ip_epilogue); - WARN_ON(err); + if (im->call_orig) { if (IS_ENABLED(CONFIG_TASKS_RCU)) call_rcu_tasks(&im->rcu, __bpf_tramp_image_put_rcu_tasks); else @@ -658,7 +662,7 @@ static void bpf_tramp_image_put(struct bpf_tramp_image *im) call_rcu_tasks_trace(&im->rcu, __bpf_tramp_image_put_rcu_tasks); } -static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size) +static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size, int nr_progs) { struct bpf_tramp_image *im; struct bpf_ksym *ksym; @@ -669,6 +673,10 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size) if (!im) goto out; + im->skips = kzalloc_objs(*im->skips, nr_progs); + if (!im->skips) + goto out_free_im; + err = bpf_jit_charge_modmem(size); if (err) goto out_free_im; @@ -695,6 +703,7 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size) out_uncharge: bpf_jit_uncharge_modmem(size); out_free_im: + kfree(im->skips); kfree(im); out: return ERR_PTR(err); @@ -771,11 +780,12 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut goto out; } - im = bpf_tramp_image_alloc(tr->key, size); + im = bpf_tramp_image_alloc(tr->key, size, total); if (IS_ERR(im)) { err = PTR_ERR(im); goto out; } + im->call_orig = tr->flags & BPF_TRAMP_F_CALL_ORIG; err = arch_prepare_bpf_trampoline(im, im->image, im->image + size, &tr->func.model, tr->flags, tnodes, @@ -806,8 +816,14 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut #endif out_free: - if (err) + if (err) { bpf_tramp_image_free(im); + } else { + /* track the image until it is freed, for bpf_trampoline_skip_prog() */ + refcount_inc(&tr->refcnt); + im->tr = tr; + list_add(&im->list, &tr->images); + } out: /* If any error happens, restore previous flags */ if (err) @@ -907,6 +923,7 @@ static int bpf_trampoline_add_prog(struct bpf_trampoline *tr, } hlist_add_head(&node->tramp_hlist, prog_list); + node->link->prog->aux->tramp_linked = true; if (kind == BPF_TRAMP_FSESSION) { tr->progs_cnt[BPF_TRAMP_FENTRY]++; fexit = fsession_exit(node); @@ -920,6 +937,41 @@ static int bpf_trampoline_add_prog(struct bpf_trampoline *tr, return 0; } +/* + * Patch the nop in front of a prog call to a jump over it. A task can be + * preempted anywhere in the image, so archs that need several instructions for + * a jump of any range patch a single near branch here instead. + */ +int __weak arch_bpf_trampoline_skip(void *nop, void *target) +{ + return bpf_arch_text_poke(nop, BPF_MOD_NOP, BPF_MOD_JUMP, NULL, target); +} + +/* + * prog was detached and can be freed, but tasks may still be running in images + * that call it, sleeping in an earlier prog for example. They can be in any + * image that is not freed yet, not only in cur_image, so patch all of them to + * jump over prog. + */ +static void bpf_trampoline_skip_prog(struct bpf_trampoline *tr, struct bpf_prog *prog) +{ + struct bpf_tramp_image *im; + int i, err; + + list_for_each_entry(im, &tr->images, list) { + for (i = 0; i < im->nr_skips; i++) { + struct bpf_tramp_skip *skip = &im->skips[i]; + + if (skip->prog != prog) + continue; + err = arch_bpf_trampoline_skip(skip->nop, skip->target); + WARN_ON_ONCE(err); + /* not a nop anymore, and prog's address can be reused */ + skip->prog = NULL; + } + } +} + static void bpf_trampoline_remove_prog(struct bpf_trampoline *tr, struct bpf_tramp_node *node) { @@ -937,6 +989,7 @@ static void bpf_trampoline_remove_prog(struct bpf_trampoline *tr, } hlist_del_init(&node->tramp_hlist); tr->progs_cnt[kind]--; + bpf_trampoline_skip_prog(tr, node->link->prog); } static int __bpf_trampoline_link_prog(struct bpf_tramp_node *node, @@ -1245,11 +1298,9 @@ void bpf_trampoline_put(struct bpf_trampoline *tr) if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[i]))) goto out; - /* This code will be executed even when the last bpf_tramp_image - * is alive. All progs are detached from the trampoline and the - * trampoline image is patched with jmp into epilogue to skip - * fexit progs. The fentry-only trampoline will be freed via - * multiple rcu callbacks. + /* + * All progs are detached and the last image has been freed, images + * hold a reference on the trampoline until then. */ hlist_del(&tr->hlist_key); hlist_del(&tr->hlist_ip); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 72a3f5998dd2..5f874979b8d7 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -567,7 +567,7 @@ static int stack_slot_obj_get_spi(struct bpf_verifier_env *env, struct bpf_reg_s } off = reg->var_off.value; - if (off % BPF_REG_SIZE) { + if (off >= 0 || off % BPF_REG_SIZE) { verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off); return -EINVAL; } @@ -1864,6 +1864,7 @@ static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm) offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type)); reg->id = 0; reg->parent_id = 0; + reg->map_uid = 0; ___mark_reg_known(reg, imm); } @@ -1925,17 +1926,18 @@ static void refine_map_lookup_value(struct bpf_reg_state *reg) if (map->inner_map_meta) { reg->type = CONST_PTR_TO_MAP | maybe_null; reg->map_ptr = map->inner_map_meta; - /* transfer reg's id which is unique for every map_lookup_elem + /* + * transfer reg's id which is unique for every map_lookup_elem * as UID of the inner map. */ - if (btf_record_has_field(map->inner_map_meta->record, - BPF_TIMER | BPF_WORKQUEUE | BPF_TASK_WORK)) - reg->map_uid = reg->id; + reg->map_uid = reg->id; } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { reg->type = PTR_TO_XDP_SOCK | maybe_null; + reg->map_uid = 0; } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP || map->map_type == BPF_MAP_TYPE_SOCKHASH) { reg->type = PTR_TO_SOCKET | maybe_null; + reg->map_uid = 0; } } @@ -3042,6 +3044,8 @@ static int check_subprogs(struct bpf_verifier_env *env) subprog[cur_subprog].exit_idx = i; goto next; } + if (insn_is_gotox(&insn[i])) + goto next; off = i + bpf_jmp_offset(&insn[i]) + 1; if (off < subprog_start || off >= subprog_end) { verbose(env, "jump out of range from insn %d to %d\n", i, off); @@ -3061,7 +3065,8 @@ static int check_subprogs(struct bpf_verifier_env *env) */ if (code != (BPF_JMP | BPF_EXIT) && code != (BPF_JMP32 | BPF_JA) && - code != (BPF_JMP | BPF_JA)) { + code != (BPF_JMP | BPF_JA) && + !insn_is_gotox(&insn[i])) { verbose(env, "last insn is not an exit or jmp\n"); bpf_diag_program_structure( env, i, "subprogram can fall through", @@ -3582,7 +3587,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, save_register_state(env, state, spi, reg, size); /* Break the relation on a narrowing spill. */ if (!reg_value_fits) - state->stack[spi].spilled_ptr.id = 0; + clear_scalar_id(&state->stack[spi].spilled_ptr); } else if (!reg && !(off % BPF_REG_SIZE) && is_bpf_st_mem(insn) && env->bpf_capable) { struct bpf_reg_state *tmp_reg = &env->fake_reg[0]; @@ -6453,6 +6458,15 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b return -EACCES; } + if (rdonly_untrusted && !env->allow_ptr_leaks) { + verbose(env, "%s access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", + reg_type_str(env, reg->type)); + bpf_diag_policy(env, insn_idx, "read from untrusted read-only memory", + "the access requires CAP_PERFMON", + "Load the program with CAP_PERFMON, or avoid dereferencing untrusted pointers."); + return -EPERM; + } + /* * Accesses to untrusted PTR_TO_MEM are done through probe * instructions, hence no need to check bounds in that case. @@ -9736,6 +9750,16 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, if (check_mem_reg(env, reg, argno, arg->mem_size, BPF_READ | BPF_WRITE, NULL, NULL)) return -EINVAL; + /* + * PTR_TO_PACKET get passed as PTR_TO_MEM, preventing + * us from adjusting bounds tracking info. + */ + if ((reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg)) && + sub->changes_pkt_data) { + bpf_log(log, "%s is a packet pointer, but func#%d may change packet data\n", + reg_arg_name(env, argno), subprog); + return -EINVAL; + } if (!(arg->arg_type & PTR_MAYBE_NULL) && (type_may_be_null(reg->type) || bpf_register_is_null(reg))) { bpf_log(log, "%s is expected to be non-NULL\n", @@ -9919,6 +9943,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err == -EFAULT) return err; if (bpf_subprog_is_global(env, subprog)) { + struct bpf_func_info_aux *sub_aux = subprog_aux(env, subprog); const char *sub_name = bpf_subprog_name(env, subprog); const char *operation; bool returns_void; @@ -9950,11 +9975,10 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (env->log.level & BPF_LOG_LEVEL) verbose(env, "Func#%d ('%s') is global and assumed valid.\n", subprog, sub_name); + sub_aux->called[in_sleepable_context(env)] = true; returns_void = subprog_returns_void(env, subprog); if (env->subprog_info[subprog].changes_pkt_data) clear_all_pkt_pointers(env); - /* mark global subprog for verifying after main prog */ - subprog_aux(env, subprog)->called = true; if (returns_void) bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED); else @@ -10036,6 +10060,7 @@ int map_set_for_each_callback_args(struct bpf_verifier_env *env, __mark_reg_known_zero(&callee->regs[BPF_REG_3]); callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr; callee->regs[BPF_REG_3].map_uid = caller->regs[BPF_REG_1].map_uid; + callee->regs[BPF_REG_3].id = ++env->id_gen; /* pointer to stack or null */ callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3]; @@ -10132,6 +10157,7 @@ static int set_timer_callback_state(struct bpf_verifier_env *env, __mark_reg_known_zero(&callee->regs[BPF_REG_3]); callee->regs[BPF_REG_3].map_ptr = map_ptr; callee->regs[BPF_REG_3].map_uid = map_uid; + callee->regs[BPF_REG_3].id = ++env->id_gen; /* unused */ bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); @@ -10250,6 +10276,7 @@ static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env, __mark_reg_known_zero(&callee->regs[BPF_REG_3]); callee->regs[BPF_REG_3].map_ptr = map_ptr; callee->regs[BPF_REG_3].map_uid = map_uid; + callee->regs[BPF_REG_3].id = ++env->id_gen; /* unused */ bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); @@ -10772,11 +10799,7 @@ int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id, /* Check if we're in a sleepable context. */ static inline bool in_sleepable_context(struct bpf_verifier_env *env) { - return !env->cur_state->active_rcu_locks && - !env->cur_state->active_preempt_locks && - !env->cur_state->active_locks && - !env->cur_state->active_irq_id && - in_sleepable(env); + return !in_rcu_cs(env); } static const char *non_sleepable_context_description(struct bpf_verifier_env *env) @@ -11356,6 +11379,11 @@ static bool is_kfunc_destructive(struct bpf_call_arg_meta *meta) return meta->kfunc_flags & KF_DESTRUCTIVE; } +static bool is_kfunc_perfmon(struct bpf_call_arg_meta *meta) +{ + return meta->kfunc_flags & KF_PERFMON; +} + static bool is_kfunc_rcu(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_RCU; @@ -13834,6 +13862,15 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return -EACCES; } + if (is_kfunc_perfmon(&meta) && !env->allow_ptr_leaks) { + verbose(env, "%s is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", + func_name); + operation = bpf_diag_fmt(env, "kfunc %s", func_name); + bpf_diag_policy(env, insn_idx, operation, "the kfunc requires CAP_PERFMON", + "Load the program with CAP_PERFMON, or avoid the kfunc."); + return -EPERM; + } + sleepable = bpf_is_kfunc_sleepable(&meta); if (sleepable && !in_sleepable(env)) { verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name); @@ -14789,7 +14826,15 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn "Tighten the scalar bounds before the arithmetic so the resulting pointer remains within the allowed range."); return -EINVAL; } - reg_bounds_sync(dst_reg); + /* + * A packet pointer that keeps its id or range is checked against a + * range set from the checked pointer's umax, so var_off must not tighten + * its umax. r32 must still match var_off for reg_bounds_sanity_check(). + */ + if (reg_is_pkt_pointer(dst_reg) && (known || dst_reg->range > 0)) + __update_reg32_bounds(dst_reg); + else + reg_bounds_sync(dst_reg); bounds_ret = sanitize_check_bounds(env, insn, dst_reg); if (bounds_ret == -EACCES) return bounds_ret; @@ -15704,6 +15749,7 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; struct bpf_reg_state *ptr_reg = NULL, off_reg = {0}; bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64); + struct bpf_insn_aux_data *aux = cur_aux(env); u8 opcode = BPF_OP(insn->code); int err; @@ -15715,12 +15761,23 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, /* Case where at least one operand is an arena. */ if (dst_reg->type == PTR_TO_ARENA || (src_reg && src_reg->type == PTR_TO_ARENA)) { - struct bpf_insn_aux_data *aux = cur_aux(env); if (dst_reg->type != PTR_TO_ARENA) *dst_reg = *src_reg; if (BPF_CLASS(insn->code) == BPF_ALU64) { + /* + * Only arena pointers set needs_zext, but doing so + * modifies the instruction at fixup time to an ALU32 + * and makes it unsuitable for 64-bit scalar args. We + * prevent zext from being set if the instruction has + * been previously called with non-arena registers. + */ + if (aux->prevent_zext) { + verbose(env, "same insn cannot be used with and without arena pointer\n"); + return -EINVAL; + } + /* * 32-bit operations zero upper bits automatically. * 64-bit operations need to be converted to 32. @@ -15733,6 +15790,16 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, return 0; } + /* Prevent the instruction from being used with arena pointers (see above). */ + if (env->prog->aux->arena && BPF_CLASS(insn->code) == BPF_ALU64) { + if (aux->needs_zext) { + verbose(env, "same insn cannot be used with and without arena pointer\n"); + return -EINVAL; + } + + aux->prevent_zext = true; + } + if (dst_reg->type != SCALAR_VALUE) ptr_reg = dst_reg; @@ -19421,13 +19488,14 @@ static void free_states(struct bpf_verifier_env *env) } } -static int do_check_common(struct bpf_verifier_env *env, int subprog) +static int do_check_common(struct bpf_verifier_env *env, int subprog, bool is_sleepable) { bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); struct bpf_subprog_info *sub = subprog_info(env, subprog); struct bpf_prog_aux *aux = env->prog->aux; struct bpf_verifier_state *state; struct bpf_reg_state *regs; + u32 old_insns_total = sub->insns_total; u32 insn_processed = env->insn_processed; int ret, i; @@ -19440,7 +19508,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) state->curframe = 0; state->speculative = false; state->branches = 1; - state->in_sleepable = env->prog->sleepable; + state->in_sleepable = is_sleepable; state->frame[0] = kzalloc_obj(struct bpf_func_state, GFP_KERNEL_ACCOUNT); if (!state->frame[0]) { kfree(state); @@ -19581,8 +19649,10 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) * not accounted as callees by account_current_path(). * Accumulate their total counts as total counts of the main or * global subprog hosting the async call. + * Start from the saved total of earlier contexts: adding to the current + * total would count this pass's synchronous paths twice. */ - env->subprog_info[subprog].insns_total = env->insn_processed - insn_processed; + sub->insns_total = old_insns_total + (env->insn_processed - insn_processed); return ret; } @@ -19610,14 +19680,19 @@ static int do_check_subprogs(struct bpf_verifier_env *env) { struct bpf_prog_aux *aux = env->prog->aux; struct bpf_func_info_aux *sub_aux; - int i, ret, new_cnt; + int context, i, ret, new_cnt; if (!aux->func_info) return 0; - /* exception callback is presumed to be always called */ - if (env->exception_callback_subprog) - subprog_aux(env, env->exception_callback_subprog)->called = true; + /* + * Callbacks cannot throw, so the exception callback always runs in the + * main program's context. It is presumed to be always called. + */ + if (env->exception_callback_subprog) { + sub_aux = subprog_aux(env, env->exception_callback_subprog); + sub_aux->called[env->prog->sleepable] = true; + } again: new_cnt = 0; @@ -19626,29 +19701,28 @@ static int do_check_subprogs(struct bpf_verifier_env *env) continue; sub_aux = subprog_aux(env, i); - if (!sub_aux->called || sub_aux->verified) - continue; + for (context = 0; context < ARRAY_SIZE(sub_aux->called); context++) { + if (!sub_aux->called[context] || sub_aux->verified[context]) + continue; - env->insn_idx = env->subprog_info[i].start; - WARN_ON_ONCE(env->insn_idx == 0); - ret = do_check_common(env, i); - if (ret) { - return ret; - } else if (env->log.level & BPF_LOG_LEVEL) { - verbose(env, "Func#%d ('%s') is safe for any args that match its prototype\n", - i, bpf_subprog_name(env, i)); + env->insn_idx = env->subprog_info[i].start; + WARN_ON_ONCE(env->insn_idx == 0); + ret = do_check_common(env, i, context); + if (ret) + return ret; + if (env->log.level & BPF_LOG_LEVEL) + verbose(env, "Func#%d ('%s') is safe for any args " + "that match its prototype\n", + i, bpf_subprog_name(env, i)); + + sub_aux->verified[context] = true; + new_cnt++; } - - /* We verified new global subprog, it might have called some - * more global subprogs that we haven't verified yet, so we - * need to do another pass over subprogs to verify those. - */ - sub_aux->verified = true; - new_cnt++; } - /* We can't loop forever as we verify at least one global subprog on - * each pass. + /* + * We can't loop forever as each pass verifies at least one new context, + * and there are only two contexts per global subprog. */ if (new_cnt) goto again; @@ -19661,7 +19735,7 @@ static int do_check_main(struct bpf_verifier_env *env) int ret; env->insn_idx = 0; - ret = do_check_common(env, 0); + ret = do_check_common(env, 0, env->prog->sleepable); if (!ret) env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; return ret; @@ -21170,6 +21244,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, ret = bpf_diag_init(env); if (ret) goto err_prep; + if (env->prog->insnsi[env->prog->len - 1].code == (BPF_LD | BPF_IMM | BPF_DW)) { + verbose(env, "invalid bpf_ld_imm64 insn\n"); + ret = -EINVAL; + goto err_prep; + } if (env->signature) { ret = bpf_prog_calc_tag(env->prog); if (ret < 0) @@ -21245,6 +21324,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; + /* Apply CO-RE before validating the program's instruction layout. */ + ret = bpf_check_core_relo(env, attr, uattr); + if (ret < 0) + goto skip_full_check; + /* Discover all subprograms before validating their layout and BTF. */ ret = add_subprogs(env); if (ret < 0) @@ -21254,7 +21338,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; - /* Validate BTF against the complete subprogram layout and apply CO-RE. */ + /* Validate BTF against the complete subprogram layout. */ ret = bpf_check_btf_info(env, attr, uattr); if (ret < 0) goto skip_full_check; diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 2538faac9aba..3f52717c1965 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -1591,10 +1591,11 @@ static int remote_partition_enable(struct cpuset *cs, int new_prs, * above it or remote partition root underneath it is not allowed. */ compute_excpus(cs, tmp->new_cpus); - WARN_ON_ONCE(cpumask_intersects(tmp->new_cpus, subpartitions_cpus)); if (!cpumask_intersects(tmp->new_cpus, cpu_active_mask) || cpumask_subset(top_cpuset.effective_cpus, tmp->new_cpus)) return PERR_INVCPUS; + if (cpumask_intersects(tmp->new_cpus, subpartitions_cpus)) + return PERR_NOCPUS; if (((new_prs == PRS_ISOLATED) && !isolated_cpus_can_update(tmp->new_cpus, NULL)) || prstate_housekeeping_conflict(new_prs, tmp->new_cpus)) diff --git a/kernel/cgroup/pids.c b/kernel/cgroup/pids.c index ecbb839d2acb..78cdc0558d0c 100644 --- a/kernel/cgroup/pids.c +++ b/kernel/cgroup/pids.c @@ -253,6 +253,11 @@ static void pids_event(struct pids_cgroup *pids_forking, } if (!cgroup_subsys_on_dfl(pids_cgrp_subsys) || cgrp_dfl_root.flags & CGRP_ROOT_PIDS_LOCAL_EVENTS) { + /* + * pids.events reports the local counter on legacy hierarchies + * and when pids_localevents is enabled. + */ + cgroup_file_notify(&p->events_file); cgroup_file_notify(&p->events_local_file); return; } diff --git a/kernel/events/core.c b/kernel/events/core.c index db7b76d6b68a..7846d70be57f 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -3764,6 +3764,9 @@ static void perf_ctx_sched_task_cb(struct perf_event_context *ctx, list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) { cpc = this_cpc(pmu_ctx->pmu); + if (cpc->task_epc != pmu_ctx) + continue; + if (cpc->sched_cb_usage && pmu_ctx->pmu->sched_task) pmu_ctx->pmu->sched_task(pmu_ctx, task, sched_in); } @@ -3914,7 +3917,7 @@ static void __perf_pmu_sched_task(struct perf_cpu_pmu_context *cpc, perf_ctx_lock(cpuctx, cpuctx->task_ctx); perf_pmu_disable(pmu); - pmu->sched_task(cpc->task_epc, task, sched_in); + pmu->sched_task(&cpc->epc, task, sched_in); perf_pmu_enable(pmu); perf_ctx_unlock(cpuctx, cpuctx->task_ctx); @@ -3924,15 +3927,17 @@ static void perf_pmu_sched_task(struct task_struct *prev, struct task_struct *next, bool sched_in) { - struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); struct perf_cpu_pmu_context *cpc, *cpc2; - /* cpuctx->task_ctx will be handled in perf_event_context_sched_in/out */ - if (prev == next || cpuctx->task_ctx) + if (prev == next) return; - list_for_each_entry_safe(cpc, cpc2, this_cpu_ptr(&sched_cb_list), sched_cb_entry) + list_for_each_entry_safe(cpc, cpc2, this_cpu_ptr(&sched_cb_list), sched_cb_entry) { + if (cpc->task_epc) + continue; + __perf_pmu_sched_task(cpc, sched_in ? next : prev, sched_in); + } } static void perf_event_switch(struct task_struct *task, @@ -5454,6 +5459,8 @@ attach_task_ctx_data(struct task_struct *task, struct kmem_cache *ctx_cache, } if (refcount_inc_not_zero(&old->refcount)) { + if (global) + old->global = true; free_perf_ctx_data(cd); /* unused */ return 0; } @@ -7619,9 +7626,11 @@ static void perf_sigtrap(struct perf_event *event) { /* * Both perf_pending_task() and perf_pending_irq() can race with the - * task exiting. + * task exiting or exec-ing. We can determine if such a race has + * occurred by checking if perf_event_exit_task(), which will set + * ctx->task to TASK_TOMBSTONE, has already been called. */ - if (current->flags & PF_EXITING) + if (event->ctx->task == TASK_TOMBSTONE) return; /* @@ -8122,10 +8131,15 @@ static void __perf_event_header__init_id(struct perf_sample_data *data, } } -void perf_event_header__init_id(struct perf_event_header *header, - struct perf_sample_data *data, - struct perf_event *event) +void perf_event_header__init(struct perf_event_header *header, + struct perf_sample_data *data, + u32 type, u16 misc, u16 size, + struct perf_event *event) { + header->type = type; + header->misc = misc; + header->size = size; + if (event->attr.sample_id_all) { header->size += event->id_header_size; __perf_event_header__init_id(data, event, event->attr.sample_type); @@ -8962,17 +8976,16 @@ perf_event_read_event(struct perf_event *event, struct perf_output_handle handle; struct perf_sample_data sample; struct perf_read_event read_event = { - .header = { - .type = PERF_RECORD_READ, - .misc = 0, - .size = sizeof(read_event) + event->read_size, - }, .pid = perf_event_pid(event, task), .tid = perf_event_tid(event, task), }; int ret; - perf_event_header__init_id(&read_event.header, &sample, event); + perf_event_header__init(&read_event.header, &sample, + PERF_RECORD_READ, + /* misc= */ 0, + sizeof(read_event) + event->read_size, + event); ret = perf_output_begin(&handle, &sample, event, read_event.header.size); if (ret) return; @@ -9213,6 +9226,7 @@ struct perf_task_event { u32 ptid; u64 time; } event_id; + int new; }; static int perf_event_task_match(struct perf_event *event) @@ -9229,17 +9243,21 @@ static void perf_event_task_output(struct perf_event *event, struct perf_output_handle handle; struct perf_sample_data sample; struct task_struct *task = task_event->task; - int ret, size = task_event->event_id.header.size; + int ret; if (!perf_event_task_match(event)) return; - perf_event_header__init_id(&task_event->event_id.header, &sample, event); + perf_event_header__init(&task_event->event_id.header, &sample, + task_event->new ? PERF_RECORD_FORK : PERF_RECORD_EXIT, + /* misc= */ 0, + sizeof(task_event->event_id), + event); ret = perf_output_begin(&handle, &sample, event, task_event->event_id.header.size); if (ret) - goto out; + return; task_event->event_id.pid = perf_event_pid(event, task); task_event->event_id.tid = perf_event_tid(event, task); @@ -9261,8 +9279,6 @@ static void perf_event_task_output(struct perf_event *event, perf_event__output_id_sample(event, &handle, &sample); perf_output_end(&handle); -out: - task_event->event_id.header.size = size; } static void perf_event_task(struct task_struct *task, @@ -9279,18 +9295,7 @@ static void perf_event_task(struct task_struct *task, task_event = (struct perf_task_event){ .task = task, .task_ctx = task_ctx, - .event_id = { - .header = { - .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT, - .misc = 0, - .size = sizeof(task_event.event_id), - }, - /* .pid */ - /* .ppid */ - /* .tid */ - /* .ptid */ - /* .time */ - }, + .new = new, }; perf_iterate_sb(perf_event_task_output, @@ -9367,6 +9372,7 @@ struct perf_comm_event { u32 pid; u32 tid; } event_id; + bool exec; }; static int perf_event_comm_match(struct perf_event *event) @@ -9380,18 +9386,21 @@ static void perf_event_comm_output(struct perf_event *event, struct perf_comm_event *comm_event = data; struct perf_output_handle handle; struct perf_sample_data sample; - int size = comm_event->event_id.header.size; int ret; if (!perf_event_comm_match(event)) return; - perf_event_header__init_id(&comm_event->event_id.header, &sample, event); + perf_event_header__init(&comm_event->event_id.header, &sample, + PERF_RECORD_COMM, + comm_event->exec ? PERF_RECORD_MISC_COMM_EXEC : 0, + sizeof(comm_event->event_id) + comm_event->comm_size, + event); ret = perf_output_begin(&handle, &sample, event, comm_event->event_id.header.size); if (ret) - goto out; + return; comm_event->event_id.pid = perf_event_pid(event, comm_event->task); comm_event->event_id.tid = perf_event_tid(event, comm_event->task); @@ -9403,8 +9412,6 @@ static void perf_event_comm_output(struct perf_event *event, perf_event__output_id_sample(event, &handle, &sample); perf_output_end(&handle); -out: - comm_event->event_id.header.size = size; } static void perf_event_comm_event(struct perf_comm_event *comm_event) @@ -9419,8 +9426,6 @@ static void perf_event_comm_event(struct perf_comm_event *comm_event) comm_event->comm = comm; comm_event->comm_size = size; - comm_event->event_id.header.size = sizeof(comm_event->event_id) + size; - perf_iterate_sb(perf_event_comm_output, comm_event, NULL); @@ -9437,15 +9442,8 @@ void perf_event_comm(struct task_struct *task, bool exec) .task = task, /* .comm */ /* .comm_size */ - .event_id = { - .header = { - .type = PERF_RECORD_COMM, - .misc = exec ? PERF_RECORD_MISC_COMM_EXEC : 0, - /* .size */ - }, - /* .pid */ - /* .tid */ - }, + /* .event_id */ + .exec = exec, }; perf_event_comm_event(&comm_event); @@ -9479,18 +9477,20 @@ static void perf_event_namespaces_output(struct perf_event *event, struct perf_namespaces_event *namespaces_event = data; struct perf_output_handle handle; struct perf_sample_data sample; - u16 header_size = namespaces_event->event_id.header.size; int ret; if (!perf_event_namespaces_match(event)) return; - perf_event_header__init_id(&namespaces_event->event_id.header, - &sample, event); + perf_event_header__init(&namespaces_event->event_id.header, &sample, + PERF_RECORD_NAMESPACES, + /* misc= */ 0, + sizeof(namespaces_event->event_id), + event); ret = perf_output_begin(&handle, &sample, event, namespaces_event->event_id.header.size); if (ret) - goto out; + return; namespaces_event->event_id.pid = perf_event_pid(event, namespaces_event->task); @@ -9502,8 +9502,6 @@ static void perf_event_namespaces_output(struct perf_event *event, perf_event__output_id_sample(event, &handle, &sample); perf_output_end(&handle); -out: - namespaces_event->event_id.header.size = header_size; } static void perf_fill_ns_link_info(struct perf_ns_link_info *ns_link_info, @@ -9534,11 +9532,7 @@ void perf_event_namespaces(struct task_struct *task) namespaces_event = (struct perf_namespaces_event){ .task = task, .event_id = { - .header = { - .type = PERF_RECORD_NAMESPACES, - .misc = 0, - .size = sizeof(namespaces_event.event_id), - }, + /* .header */ /* .pid */ /* .tid */ .nr_namespaces = NR_NAMESPACES, @@ -9606,18 +9600,19 @@ static void perf_event_cgroup_output(struct perf_event *event, void *data) struct perf_cgroup_event *cgroup_event = data; struct perf_output_handle handle; struct perf_sample_data sample; - u16 header_size = cgroup_event->event_id.header.size; int ret; + u16 size = sizeof(cgroup_event->event_id) + cgroup_event->path_size; if (!perf_event_cgroup_match(event)) return; - perf_event_header__init_id(&cgroup_event->event_id.header, - &sample, event); + perf_event_header__init(&cgroup_event->event_id.header, &sample, + PERF_RECORD_CGROUP, /* misc= */ 0, size, + event); ret = perf_output_begin(&handle, &sample, event, cgroup_event->event_id.header.size); if (ret) - goto out; + return; perf_output_put(&handle, cgroup_event->event_id); __output_copy(&handle, cgroup_event->path, cgroup_event->path_size); @@ -9625,8 +9620,6 @@ static void perf_event_cgroup_output(struct perf_event *event, void *data) perf_event__output_id_sample(event, &handle, &sample); perf_output_end(&handle); -out: - cgroup_event->event_id.header.size = header_size; } static void perf_event_cgroup(struct cgroup *cgrp) @@ -9641,11 +9634,6 @@ static void perf_event_cgroup(struct cgroup *cgrp) cgroup_event = (struct perf_cgroup_event){ .event_id = { - .header = { - .type = PERF_RECORD_CGROUP, - .misc = 0, - .size = sizeof(cgroup_event.event_id), - }, .id = cgroup_id(cgrp), }, }; @@ -9668,7 +9656,6 @@ static void perf_event_cgroup(struct cgroup *cgrp) while (!IS_ALIGNED(size, sizeof(u64))) cgroup_event.path[size++] = '\0'; - cgroup_event.event_id.header.size += size; cgroup_event.path_size = size; perf_iterate_sb(perf_event_cgroup_output, @@ -9724,38 +9711,40 @@ static void perf_event_mmap_output(struct perf_event *event, struct perf_mmap_event *mmap_event = data; struct perf_output_handle handle; struct perf_sample_data sample; - int size = mmap_event->event_id.header.size; - u32 type = mmap_event->event_id.header.type; - bool use_build_id; + int size = sizeof(mmap_event->event_id) + mmap_event->file_size; + u32 type = PERF_RECORD_MMAP; + u16 misc = PERF_RECORD_MISC_USER; + bool use_build_id = false; int ret; if (!perf_event_mmap_match(event, data)) return; if (event->attr.mmap2) { - mmap_event->event_id.header.type = PERF_RECORD_MMAP2; - mmap_event->event_id.header.size += sizeof(mmap_event->maj); - mmap_event->event_id.header.size += sizeof(mmap_event->min); - mmap_event->event_id.header.size += sizeof(mmap_event->ino); - mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation); - mmap_event->event_id.header.size += sizeof(mmap_event->prot); - mmap_event->event_id.header.size += sizeof(mmap_event->flags); + type = PERF_RECORD_MMAP2; + size += sizeof(mmap_event->maj); + size += sizeof(mmap_event->min); + size += sizeof(mmap_event->ino); + size += sizeof(mmap_event->ino_generation); + size += sizeof(mmap_event->prot); + size += sizeof(mmap_event->flags); + use_build_id = event->attr.build_id && mmap_event->build_id_size; + if (use_build_id) + misc |= PERF_RECORD_MISC_MMAP_BUILD_ID; } + if (!(mmap_event->vma->vm_flags & VM_EXEC)) + misc |= PERF_RECORD_MISC_MMAP_DATA; - perf_event_header__init_id(&mmap_event->event_id.header, &sample, event); + perf_event_header__init(&mmap_event->event_id.header, &sample, + type, misc, size, event); ret = perf_output_begin(&handle, &sample, event, mmap_event->event_id.header.size); if (ret) - goto out; + return; mmap_event->event_id.pid = perf_event_pid(event, current); mmap_event->event_id.tid = perf_event_tid(event, current); - use_build_id = event->attr.build_id && mmap_event->build_id_size; - - if (event->attr.mmap2 && use_build_id) - mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_BUILD_ID; - perf_output_put(&handle, mmap_event->event_id); if (event->attr.mmap2) { @@ -9780,9 +9769,6 @@ static void perf_event_mmap_output(struct perf_event *event, perf_event__output_id_sample(event, &handle, &sample); perf_output_end(&handle); -out: - mmap_event->event_id.header.size = size; - mmap_event->event_id.header.type = type; } static void perf_event_mmap_event(struct perf_mmap_event *mmap_event) @@ -9878,11 +9864,6 @@ static void perf_event_mmap_event(struct perf_mmap_event *mmap_event) mmap_event->prot = prot; mmap_event->flags = flags; - if (!(vma->vm_flags & VM_EXEC)) - mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA; - - mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size; - if (atomic_read(&nr_build_id_events)) build_id_parse_nofault(vma, mmap_event->build_id, &mmap_event->build_id_size); @@ -10002,11 +9983,7 @@ void perf_event_mmap(struct vm_area_struct *vma) /* .file_name */ /* .file_size */ .event_id = { - .header = { - .type = PERF_RECORD_MMAP, - .misc = PERF_RECORD_MISC_USER, - /* .size */ - }, + /* .header */ /* .pid */ /* .tid */ .start = vma->vm_start, @@ -10036,18 +10013,15 @@ void perf_event_aux_event(struct perf_event *event, unsigned long head, u64 size; u64 flags; } rec = { - .header = { - .type = PERF_RECORD_AUX, - .misc = 0, - .size = sizeof(rec), - }, .offset = head, .size = size, .flags = flags, }; int ret; - perf_event_header__init_id(&rec.header, &sample, event); + perf_event_header__init(&rec.header, &sample, + PERF_RECORD_AUX, /* misc= */ 0, sizeof(rec), + event); ret = perf_output_begin(&handle, &sample, event, rec.header.size); if (ret) @@ -10072,15 +10046,14 @@ void perf_log_lost_samples(struct perf_event *event, u64 lost) struct perf_event_header header; u64 lost; } lost_samples_event = { - .header = { - .type = PERF_RECORD_LOST_SAMPLES, - .misc = 0, - .size = sizeof(lost_samples_event), - }, .lost = lost, }; - perf_event_header__init_id(&lost_samples_event.header, &sample, event); + perf_event_header__init(&lost_samples_event.header, &sample, + PERF_RECORD_LOST_SAMPLES, + /* misc= */ 0, + sizeof(lost_samples_event), + event); ret = perf_output_begin(&handle, &sample, event, lost_samples_event.header.size); @@ -10105,6 +10078,8 @@ struct perf_switch_event { u32 next_prev_pid; u32 next_prev_tid; } event_id; + bool sched_in; + bool preempt; }; static int perf_event_switch_match(struct perf_event *event) @@ -10117,6 +10092,9 @@ static void perf_event_switch_output(struct perf_event *event, void *data) struct perf_switch_event *se = data; struct perf_output_handle handle; struct perf_sample_data sample; + u32 type; + u16 misc; + u16 size; int ret; if (!perf_event_switch_match(event)) @@ -10124,18 +10102,22 @@ static void perf_event_switch_output(struct perf_event *event, void *data) /* Only CPU-wide events are allowed to see next/prev pid/tid */ if (event->ctx->task) { - se->event_id.header.type = PERF_RECORD_SWITCH; - se->event_id.header.size = sizeof(se->event_id.header); + type = PERF_RECORD_SWITCH; + size = sizeof(se->event_id.header); } else { - se->event_id.header.type = PERF_RECORD_SWITCH_CPU_WIDE; - se->event_id.header.size = sizeof(se->event_id); + type = PERF_RECORD_SWITCH_CPU_WIDE; + size = sizeof(se->event_id); se->event_id.next_prev_pid = perf_event_pid(event, se->next_prev); se->event_id.next_prev_tid = perf_event_tid(event, se->next_prev); } + misc = se->sched_in ? 0 : PERF_RECORD_MISC_SWITCH_OUT; + if (se->preempt) + misc |= PERF_RECORD_MISC_SWITCH_OUT_PREEMPT; - perf_event_header__init_id(&se->event_id.header, &sample, event); + perf_event_header__init(&se->event_id.header, &sample, + type, misc, size, event); ret = perf_output_begin(&handle, &sample, event, se->event_id.header.size); if (ret) @@ -10161,22 +10143,11 @@ static void perf_event_switch(struct task_struct *task, switch_event = (struct perf_switch_event){ .task = task, .next_prev = next_prev, - .event_id = { - .header = { - /* .type */ - .misc = sched_in ? 0 : PERF_RECORD_MISC_SWITCH_OUT, - /* .size */ - }, - /* .next_prev_pid */ - /* .next_prev_tid */ - }, + /* .event_id */ + .sched_in = sched_in, + .preempt = !sched_in && task_is_runnable(task), }; - if (!sched_in && task_is_runnable(task)) { - switch_event.event_id.header.misc |= - PERF_RECORD_MISC_SWITCH_OUT_PREEMPT; - } - perf_iterate_sb(perf_event_switch_output, &switch_event, NULL); } @@ -10196,20 +10167,17 @@ static void perf_log_throttle(struct perf_event *event, int enable) u64 id; u64 stream_id; } throttle_event = { - .header = { - .type = PERF_RECORD_THROTTLE, - .misc = 0, - .size = sizeof(throttle_event), - }, .time = perf_event_clock(event), .id = primary_event_id(event), .stream_id = event->id, }; - if (enable) - throttle_event.header.type = PERF_RECORD_UNTHROTTLE; - - perf_event_header__init_id(&throttle_event.header, &sample, event); + perf_event_header__init(&throttle_event.header, &sample, + enable ? PERF_RECORD_UNTHROTTLE + : PERF_RECORD_THROTTLE, + /* misc= */ 0, + sizeof(throttle_event), + event); ret = perf_output_begin(&handle, &sample, event, throttle_event.header.size); @@ -10248,12 +10216,14 @@ static void perf_event_ksymbol_output(struct perf_event *event, void *data) struct perf_output_handle handle; struct perf_sample_data sample; int ret; + u16 size = sizeof(ksymbol_event->event_id) + ksymbol_event->name_len; if (!perf_event_ksymbol_match(event)) return; - perf_event_header__init_id(&ksymbol_event->event_id.header, - &sample, event); + perf_event_header__init(&ksymbol_event->event_id.header, &sample, + PERF_RECORD_KSYMBOL, /* misc= */ 0, size, + event); ret = perf_output_begin(&handle, &sample, event, ksymbol_event->event_id.header.size); if (ret) @@ -10294,11 +10264,6 @@ void perf_event_ksymbol(u16 ksym_type, u64 addr, u32 len, bool unregister, .name = name, .name_len = name_len, .event_id = { - .header = { - .type = PERF_RECORD_KSYMBOL, - .size = sizeof(ksymbol_event.event_id) + - name_len, - }, .addr = addr, .len = len, .ksym_type = ksym_type, @@ -10342,8 +10307,11 @@ static void perf_event_bpf_output(struct perf_event *event, void *data) if (!perf_event_bpf_match(event)) return; - perf_event_header__init_id(&bpf_event->event_id.header, - &sample, event); + perf_event_header__init(&bpf_event->event_id.header, &sample, + PERF_RECORD_BPF_EVENT, + /* misc= */ 0, + sizeof(bpf_event->event_id), + event); ret = perf_output_begin(&handle, &sample, event, bpf_event->event_id.header.size); if (ret) @@ -10399,10 +10367,6 @@ void perf_event_bpf_event(struct bpf_prog *prog, bpf_event = (struct perf_bpf_event){ .prog = prog, .event_id = { - .header = { - .type = PERF_RECORD_BPF_EVENT, - .size = sizeof(bpf_event.event_id), - }, .type = type, .flags = flags, .id = prog->aux->id, @@ -10430,18 +10394,23 @@ static void perf_callchain_deferred_output(struct perf_event *event, void *data) struct perf_callchain_deferred_event *deferred_event = data; struct perf_output_handle handle; struct perf_sample_data sample; - int ret, size = deferred_event->event.header.size; + int ret; + u16 size = sizeof(deferred_event->event) + (deferred_event->trace->nr * sizeof(u64)); if (!event->attr.defer_output) return; /* XXX do we really need sample_id_all for this ??? */ - perf_event_header__init_id(&deferred_event->event.header, &sample, event); + perf_event_header__init(&deferred_event->event.header, &sample, + PERF_RECORD_CALLCHAIN_DEFERRED, + PERF_RECORD_MISC_USER, + size, + event); ret = perf_output_begin(&handle, &sample, event, deferred_event->event.header.size); if (ret) - goto out; + return; perf_output_put(&handle, deferred_event->event); for (int i = 0; i < deferred_event->trace->nr; i++) { @@ -10451,8 +10420,6 @@ static void perf_callchain_deferred_output(struct perf_event *event, void *data) perf_event__output_id_sample(event, &handle, &sample); perf_output_end(&handle); -out: - deferred_event->event.header.size = size; } static void perf_unwind_deferred_callback(struct unwind_work *work, @@ -10461,12 +10428,6 @@ static void perf_unwind_deferred_callback(struct unwind_work *work, struct perf_callchain_deferred_event deferred_event = { .trace = trace, .event = { - .header = { - .type = PERF_RECORD_CALLCHAIN_DEFERRED, - .misc = PERF_RECORD_MISC_USER, - .size = sizeof(deferred_event.event) + - (trace->nr * sizeof(u64)), - }, .cookie = cookie, .nr = trace->nr, }, @@ -10478,7 +10439,8 @@ static void perf_unwind_deferred_callback(struct unwind_work *work, struct perf_text_poke_event { const void *old_bytes; const void *new_bytes; - size_t pad; + u16 tot; + u16 pad; u16 old_len; u16 new_len; @@ -10499,13 +10461,18 @@ static void perf_event_text_poke_output(struct perf_event *event, void *data) struct perf_text_poke_event *text_poke_event = data; struct perf_output_handle handle; struct perf_sample_data sample; + u16 size = sizeof(text_poke_event->event_id) + text_poke_event->tot + text_poke_event->pad; u64 padding = 0; int ret; if (!perf_event_text_poke_match(event)) return; - perf_event_header__init_id(&text_poke_event->event_id.header, &sample, event); + perf_event_header__init(&text_poke_event->event_id.header, &sample, + PERF_RECORD_TEXT_POKE, + PERF_RECORD_MISC_KERNEL, + size, + event); ret = perf_output_begin(&handle, &sample, event, text_poke_event->event_id.header.size); @@ -10543,15 +10510,11 @@ void perf_event_text_poke(const void *addr, const void *old_bytes, text_poke_event = (struct perf_text_poke_event){ .old_bytes = old_bytes, .new_bytes = new_bytes, + .tot = tot, .pad = pad, .old_len = old_len, .new_len = new_len, .event_id = { - .header = { - .type = PERF_RECORD_TEXT_POKE, - .misc = PERF_RECORD_MISC_KERNEL, - .size = sizeof(text_poke_event.event_id) + tot + pad, - }, .addr = (unsigned long)addr, }, }; @@ -10582,13 +10545,12 @@ static void perf_log_itrace_start(struct perf_event *event) event->attach_state & PERF_ATTACH_ITRACE) return; - rec.header.type = PERF_RECORD_ITRACE_START; - rec.header.misc = 0; - rec.header.size = sizeof(rec); rec.pid = perf_event_pid(event, current); rec.tid = perf_event_tid(event, current); - perf_event_header__init_id(&rec.header, &sample, event); + perf_event_header__init(&rec.header, &sample, + PERF_RECORD_ITRACE_START, /* misc= */ 0, sizeof(rec), + event); ret = perf_output_begin(&handle, &sample, event, rec.header.size); if (ret) @@ -10613,12 +10575,10 @@ void perf_report_aux_output_id(struct perf_event *event, u64 hw_id) if (event->parent) event = event->parent; - rec.header.type = PERF_RECORD_AUX_OUTPUT_HW_ID; - rec.header.misc = 0; - rec.header.size = sizeof(rec); - rec.hw_id = hw_id; - - perf_event_header__init_id(&rec.header, &sample, event); + rec.hw_id = hw_id; + perf_event_header__init(&rec.header, &sample, + PERF_RECORD_AUX_OUTPUT_HW_ID, /* misc= */ 0, + sizeof(rec), event); ret = perf_output_begin(&handle, &sample, event, rec.header.size); if (ret) @@ -13946,7 +13906,7 @@ SYSCALL_DEFINE5(perf_event_open, if (err) return err; - if (!attr.exclude_kernel || + if (!attr.exclude_kernel || attr.text_poke || ((attr.sample_type & PERF_SAMPLE_CALLCHAIN) && !attr.exclude_callchain_kernel)) { err = perf_allow_kernel(); diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c index 1b1ffe0533e5..13bd42e5493d 100644 --- a/kernel/events/ring_buffer.c +++ b/kernel/events/ring_buffer.c @@ -246,14 +246,13 @@ __perf_output_begin(struct perf_output_handle *handle, handle->size = (1UL << page_shift) - offset; if (unlikely(have_lost)) { - lost_event.header.size = sizeof(lost_event); - lost_event.header.type = PERF_RECORD_LOST; - lost_event.header.misc = 0; lost_event.id = event->id; lost_event.lost = local_xchg(&rb->lost, 0); /* XXX mostly redundant; @data is already fully initializes */ - perf_event_header__init_id(&lost_event.header, data, event); + perf_event_header__init(&lost_event.header, data, + PERF_RECORD_LOST, /* misc= */ 0, + sizeof(lost_event), event); perf_output_put(handle, lost_event); perf_event__output_id_sample(event, handle, data); } diff --git a/kernel/exit.c b/kernel/exit.c index 424c44a42a4d..282328d2b4cf 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -551,32 +551,6 @@ void mm_update_next_owner(struct mm_struct *mm) } #endif /* CONFIG_MEMCG */ -#if defined(CONFIG_SCHED_CACHE) && defined(CONFIG_NUMA_BALANCING) -/* - * Subtract the memory footprint of the current task from - * mm. - */ -static void exit_mm_sched_cache(struct mm_struct *mm) -{ - unsigned long fp, sub; - - if (!current->total_numa_faults) - return; - /* - * No lock protection due to performance considerations. - * Make sure mm->sc_stat.footprint does not become - * negative. - */ - fp = READ_ONCE(mm->sc_stat.footprint); - sub = min(fp, current->total_numa_faults); - WRITE_ONCE(mm->sc_stat.footprint, fp - sub); -} -#else -static inline void exit_mm_sched_cache(struct mm_struct *mm) -{ -} -#endif /* CONFIG_SCHED_CACHE CONFIG_NUMA_BALANCING */ - /* * Turn us into a lazy TLB process if we * aren't already.. @@ -589,7 +563,7 @@ static void exit_mm(void) if (!mm) return; - exit_mm_sched_cache(mm); + sched_cache_exit_mm(current); mmap_read_lock(mm); mmgrab_lazy_tlb(mm); diff --git a/kernel/fork.c b/kernel/fork.c index 5ef413368912..10f2d05d816a 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1599,6 +1599,7 @@ static int copy_mm(u64 clone_flags, struct task_struct *tsk) tsk->mm = mm; tsk->active_mm = mm; + sched_cache_fork(tsk); return 0; } @@ -2602,6 +2603,7 @@ __latent_entropy struct task_struct *copy_process( bad_fork_cleanup_namespaces: exit_nsproxy_namespaces(p); bad_fork_cleanup_mm: + sched_cache_fork_cleanup(p); if (p->mm) { mm_clear_owner(p->mm, p); mmput(p->mm); diff --git a/kernel/futex/core.c b/kernel/futex/core.c index a061f54b606d..095f9fe440e3 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -213,10 +213,12 @@ static bool __futex_pivot_hash(struct mm_struct *mm, struct futex_private_hash * futex_rehash_private(fph, new); } new->state = FR_PERCPU; - scoped_guard(rcu) { - mmph->batches = get_state_synchronize_rcu(); - rcu_assign_pointer(mmph->hash, new); - } + rcu_assign_pointer(mmph->hash, new); + /* + * mmph->batches must reference a grace period which started after + * mmph->hash was assigned. See futex_ref_drop(). + */ + mmph->batches = get_state_synchronize_rcu(); kvfree_rcu(fph, rcu); return true; } diff --git a/kernel/irq/Kconfig b/kernel/irq/Kconfig index 05cba4e16dad..6923f37eaab4 100644 --- a/kernel/irq/Kconfig +++ b/kernel/irq/Kconfig @@ -150,6 +150,18 @@ config IRQ_KUNIT_TEST If unsure, say N. +config REFCOUNT_INTERRUPT_KUNIT_TEST + tristate "Test refcounted interrupt enable/disable" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + This builds the kunit tests for the refcounted interrupt + infrastructure. It verifies the correctness of single, nested, + and multiple interrupt enable/disable state changes and ensures + that the underlying reference counting mechanisms work as expected. + + If unsure, say N. + endmenu config GENERIC_IRQ_MULTI_HANDLER diff --git a/kernel/irq/Makefile b/kernel/irq/Makefile index 44c4d6fc502a..0e5df962a149 100644 --- a/kernel/irq/Makefile +++ b/kernel/irq/Makefile @@ -16,4 +16,4 @@ obj-$(CONFIG_SMP) += affinity.o obj-$(CONFIG_GENERIC_IRQ_DEBUGFS) += debugfs.o obj-$(CONFIG_GENERIC_IRQ_MATRIX_ALLOCATOR) += matrix.o obj-$(CONFIG_IRQ_KUNIT_TEST) += irq_test.o -obj-$(CONFIG_KUNIT) += refcount_interrupt_test.o +obj-$(CONFIG_REFCOUNT_INTERRUPT_KUNIT_TEST) += refcount_interrupt_test.o diff --git a/kernel/kprobes.c b/kernel/kprobes.c index 6337da5cab9e..e787e4948c8b 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c @@ -42,6 +42,7 @@ #include #include #include +#include #include #include @@ -495,14 +496,16 @@ static bool kprobe_queued(struct kprobe *p) static struct kprobe *get_optimized_kprobe(kprobe_opcode_t *addr) { int i; - struct kprobe *p = NULL; + struct kprobe *p; struct optimized_kprobe *op; /* Don't check i == 0, since that is a breakpoint case. */ - for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH / sizeof(kprobe_opcode_t); i++) + for (i = 1; i < MAX_OPTIMIZED_LENGTH / sizeof(kprobe_opcode_t); i++) { p = get_kprobe(addr - i); + /* A disabled probe can have prepared, but inactive, optinsns. */ + if (!p || !kprobe_optready(p) || kprobe_disarmed(p)) + continue; - if (p && kprobe_optready(p)) { op = container_of(p, struct optimized_kprobe, kp); if (arch_within_optimized_kprobe(op, addr)) return p; @@ -526,7 +529,8 @@ enum { OPTIMIZER_ST_FLUSHING = 2, }; -static DECLARE_COMPLETION(optimizer_completion); +/* Bumped at the end of each kprobe_optimizer() pass, under 'kprobe_mutex' */ +static unsigned long optimizer_passes; #define OPTIMIZE_DELAY 5 @@ -654,9 +658,9 @@ static void kprobe_optimizer(void) do_free_cleaned_kprobes(); } - /* Step 5: Kick optimizer again if needed. But if there is a flush requested, */ - if (completion_done(&optimizer_completion)) - complete(&optimizer_completion); + /* Step 5: Wake up flushers, and kick optimizer again if needed. */ + optimizer_passes++; + wake_up_var_locked(&optimizer_passes, &kprobe_mutex); if (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) kick_kprobe_optimizer(); /*normal kick*/ @@ -708,7 +712,8 @@ static void wait_for_kprobe_optimizer_locked(void) lockdep_assert_held(&kprobe_mutex); while (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) { - init_completion(&optimizer_completion); + unsigned long passes = optimizer_passes; + /* * Set state to OPTIMIZER_ST_FLUSHING and wake up the thread if it's * idle. If it's already kicked, it will see the state change. @@ -717,9 +722,12 @@ static void wait_for_kprobe_optimizer_locked(void) OPTIMIZER_ST_FLUSHING) != OPTIMIZER_ST_FLUSHING) wake_up(&kprobe_optimizer_wait); - mutex_unlock(&kprobe_mutex); - wait_for_completion(&optimizer_completion); - mutex_lock(&kprobe_mutex); + /* + * kprobe_optimizer() holds 'kprobe_mutex' for a whole pass, which + * this drops while sleeping, so a new count means a full pass ran. + */ + wait_var_event_mutex(&optimizer_passes, + optimizer_passes != passes, &kprobe_mutex); } } diff --git a/kernel/module/main.c b/kernel/module/main.c index d0e1e0bd2ad0..c1b34dc1e89a 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -3581,8 +3581,9 @@ static int load_module(struct load_info *info, const char __user *uargs, goto sysfs_cleanup; } - if (codetag_load_module(mod)) - goto sysfs_cleanup; + err = codetag_load_module(mod); + if (err) + goto livepatch_cleanup; /* Get rid of temporary copy. */ free_copy(info, flags); @@ -3592,6 +3593,9 @@ static int load_module(struct load_info *info, const char __user *uargs, return do_init_module(mod); + livepatch_cleanup: + if (is_livepatch_module(mod)) + free_module_elf(mod); sysfs_cleanup: mod_sysfs_teardown(mod); coming_cleanup: diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index d2479c69d71a..c13f68ab7f6e 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c @@ -408,9 +408,18 @@ int hibernation_snapshot(int platform_mode) if (error) goto Close; + error = dpm_prepare(PMSG_FREEZE); + if (error) + goto Complete; + + /* Preallocate image memory before freezing kernel threads and shutting down devices. */ + error = hibernate_preallocate_memory(); + if (error) + goto Complete; + error = freeze_kernel_threads(); if (error) - goto Close; + goto Cleanup; if (hibernation_test(TEST_FREEZER)) { @@ -422,15 +431,6 @@ int hibernation_snapshot(int platform_mode) goto Thaw; } - error = dpm_prepare(PMSG_FREEZE); - if (error) - goto Complete; - - /* Preallocate image memory before shutting down devices. */ - error = hibernate_preallocate_memory(); - if (error) - goto Complete; - console_suspend_all(); pm_restrict_gfp_mask(); @@ -464,10 +464,12 @@ int hibernation_snapshot(int platform_mode) platform_end(platform_mode); return error; - Complete: - dpm_complete(PMSG_RECOVER); Thaw: thaw_kernel_threads(); + Cleanup: + swsusp_free(); + Complete: + dpm_complete(PMSG_RECOVER); goto Close; } diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 0b846a13c628..1fe40de6ebe3 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5798,7 +5798,7 @@ void sched_tick(void) curr = rq->curr; donor = rq->donor; - psi_account_irqtime(rq, donor, NULL); + psi_account_irqtime(rq, curr, NULL); update_rq_clock(rq); hw_pressure = arch_scale_hw_pressure(cpu_of(rq)); diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c index bc4eee5bb4cb..4b08866d75f6 100644 --- a/kernel/sched/ext/cid.c +++ b/kernel/sched/ext/cid.c @@ -912,30 +912,36 @@ bool scx_cmask_empty(const struct scx_cmask *m) /** * scx_bpf_cid_topo - Copy out per-cid topology info * @cid: cid to look up - * @out__uninit: where to copy the topology info; fully written by this call + * @out: where to copy the topology info + * @out__sz: size of @out, the program's sizeof(struct scx_cid_topo) * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * - * Fill @out__uninit with the topology info for @cid. Trigger scx_error() if - * @cid is out of range. If @cid is valid but in the no-topo section, all fields - * are set to -1. All fields are also set to -1 when no cid tables have been - * published yet, which a program may observe while racing the root enable. + * Fill @out with the topology info for @cid. Trigger scx_error() if @cid is out + * of range. If @cid is valid but in the no-topo section, all fields are set to + * -1. All fields are also set to -1 when no cid tables have been published yet, + * which a program may observe while racing the root enable. + * + * The program's struct may be older or newer than the kernel's. The smaller of + * @out__sz and the kernel's size is copied and the rest of @out is set to -1. */ -__bpf_kfunc void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out__uninit, +__bpf_kfunc void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out, size_t out__sz, const struct bpf_prog_aux *aux) { + size_t len = min(out__sz, sizeof(*out)); struct scx_cid_topo *topo; struct scx_sched *sch; + /* the error cases and fields the kernel lacks read as -1 */ + memset(out, 0xff, out__sz); + guard(rcu)(); sch = scx_prog_sched(aux); topo = rcu_dereference(scx_cid_topo); - if (unlikely(!sch) || !cid_valid(sch, cid) || unlikely(!topo)) { - *out__uninit = SCX_CID_TOPO_NEG; + if (unlikely(!sch) || !cid_valid(sch, cid) || unlikely(!topo)) return; - } - *out__uninit = topo[cid]; + memcpy(out, &topo[cid], len); } __bpf_kfunc_end_defs(); diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 3219f0da0fe4..e56c3c95018f 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -447,37 +447,45 @@ static void switch_rq_lock(struct rq *from, struct rq *to) DEFINE_STATIC_KEY_FALSE(__scx_is_cid_type); /** - * scx_call_op_set_cpumask - invoke ops.set_cpumask / ops_cid.set_cmask for @task + * scx_fill_cmask_scratch - Build this cpu's arena cmask from @cpumask + * @sch: scx_sched whose scratch to fill + * @cpumask: cpus to translate into cids + * + * The scratch lives in BPF-writable arena memory and its header can't be + * trusted, so it is rewritten from kernel geometry rather than read. Caller + * must hold an rq lock so this cpu is the sole kernel writer for as long as the + * returned address is in use. + */ +static struct scx_cmask *scx_fill_cmask_scratch(struct scx_sched *sch, + const struct cpumask *cpumask) +{ + struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch); + struct scx_cmask_ref ref; + + scx_cmask_ref_init_kern(sch, kern_va, 0, num_possible_cpus(), &ref); + scx_cmask_ref_from_cpumask(&ref, cpumask); + return kern_va; +} + +/** + * scx_call_op_set_cpumask - Invoke the set_cpumask or set_cmask op for @task * @sch: scx_sched being invoked * @rq: rq to update as the currently-locked rq, or NULL * @task: task whose affinity is changing * @cpumask: new cpumask * - * For cid-form schedulers, translate @cpumask to a cmask via the per-cpu - * scratch in cid.c and dispatch through the ops_cid union view. Caller - * must hold @rq's rq lock so this_cpu_ptr is stable across the call. + * For cid-form schedulers, translate @cpumask to a cmask in the per-cpu scratch + * and dispatch through the ops_cid union view. Caller must hold @rq's rq lock. */ static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, struct task_struct *task, const struct cpumask *cpumask) { - if (scx_is_cid_type()) { - struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch); - struct scx_cmask_ref ref; - - /* - * Build the per-cpu arena cmask from kernel geometry via @ref, - * never reading its BPF-writable header. set_cmask()'s __arena - * argument takes the kernel address and the struct_ops - * trampoline rebases it into BPF's arena pointer form. The rq - * lock makes this cpu the sole kernel writer. - */ - scx_cmask_ref_init_kern(sch, kern_va, 0, num_possible_cpus(), &ref); - scx_cmask_ref_from_cpumask(&ref, cpumask); - SCX_CALL_CID_OP_TASK(sch, set_cmask, rq, task, kern_va); - } else { + if (scx_is_cid_type()) + SCX_CALL_CID_OP_TASK(sch, set_cmask, rq, task, + scx_fill_cmask_scratch(sch, cpumask)); + else SCX_CALL_OP_TASK(sch, set_cpumask, rq, task, cpumask); - } } enum scx_dsq_iter_flags { @@ -1499,27 +1507,22 @@ static inline bool task_scx_migrating(struct task_struct *p) return p->scx.sticky_cpu >= 0; } -/* - * Call ops.dequeue() if the task is in BPF custody and not migrating. - * Clears %SCX_TASK_IN_CUSTODY when the callback is invoked. - */ -static void call_task_dequeue(struct scx_sched *sch, struct rq *rq, - struct task_struct *p, u64 deq_flags) +/* Must be called under the lock serializing @p's custody transfers. */ +static bool task_leave_custody(struct task_struct *p) { if (!(p->scx.flags & SCX_TASK_IN_CUSTODY) || task_scx_migrating(p)) - return; - - if (SCX_HAS_OP(sch, dequeue)) - SCX_CALL_OP_TASK(sch, dequeue, rq, p, deq_flags); + return false; p->scx.flags &= ~SCX_TASK_IN_CUSTODY; + return true; } static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, struct task_struct *p, u64 enq_flags) { - call_task_dequeue(sch, rq, p, 0); + if (task_leave_custody(p) && SCX_HAS_OP(sch, dequeue)) + SCX_CALL_OP_TASK(sch, dequeue, rq, p, 0); /* * Only local inserts get the wakeup treatment below. Rejects kick the @@ -1705,20 +1708,28 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, if (is_rq_owned) { rq_owned_post_enq(sch, rq, dsq, p, enq_flags); } else { + bool call_dequeue = false; + /* * Global and bypass DSQs are terminal - the task leaves the - * scheduler's custody, so ops.dequeue() fires here. It can run + * scheduler's custody, so ops.dequeue() fires. It can run * without @p's rq lock (finish_dispatch() passes the dispatch * rq); that's safe because dequeue_task_scx() waits on * SCX_OPSS_DISPATCHING (see the ops_state note above) and so * can't race it. A non-terminal DSQ keeps the task in custody. + * The custody transfer happens under @dsq->lock so that later + * consumers see the flag clear; the callback runs after + * @dsq->lock is dropped because it may lock a DSQ itself. */ if (dsq->id == SCX_DSQ_GLOBAL || dsq->id == SCX_DSQ_BYPASS) - call_task_dequeue(sch, rq, p, 0); + call_dequeue = task_leave_custody(p); else p->scx.flags |= SCX_TASK_IN_CUSTODY; raw_spin_unlock(&dsq->lock); + + if (call_dequeue && SCX_HAS_OP(sch, dequeue)) + SCX_CALL_OP_TASK(sch, dequeue, rq, p, 0); } /* @@ -2141,7 +2152,12 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_ int sticky_cpu = p->scx.sticky_cpu; u64 enq_flags = core_enq_flags | rq->scx.remote_activate_enq_flags; - if (enq_flags & ENQUEUE_WAKEUP) + /* + * SCX_RQ_IN_WAKEUP promises a task_woken_scx() call once this enqueue + * returns. Only the core's wakeup path delivers one. The flags stashed + * for a remote activation may carry the wakeup bit without it. + */ + if (core_enq_flags & ENQUEUE_WAKEUP) rq->scx.flags |= SCX_RQ_IN_WAKEUP; /* @@ -2210,7 +2226,7 @@ static void ops_dequeue(struct rq *rq, struct task_struct *p, u64 deq_flags) /* * A queued task must always be in BPF scheduler's custody. If * SCX_TASK_IN_CUSTODY is clear, finish_dispatch() on another - * CPU has already passed call_task_dequeue() (which clears the + * CPU has already passed task_leave_custody() (which clears the * flag), but has not yet written SCX_OPSS_NONE. That final * store does not require this rq's lock, so retrying with * cpu_relax() is bounded: we will observe NONE (or DISPATCHING, @@ -2258,7 +2274,8 @@ static void ops_dequeue(struct rq *rq, struct task_struct *p, u64 deq_flags) * NONE but the task may still have %SCX_TASK_IN_CUSTODY set until * it is enqueued on the destination. */ - call_task_dequeue(sch, rq, p, deq_flags); + if (task_leave_custody(p) && SCX_HAS_OP(sch, dequeue)) + SCX_CALL_OP_TASK(sch, dequeue, rq, p, deq_flags); } static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_flags) @@ -2374,14 +2391,10 @@ static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_fl } void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct *p, - u64 enq_flags, struct scx_dispatch_q *src_dsq, - struct rq *dst_rq) + u64 enq_flags, struct rq *dst_rq) { struct scx_dispatch_q *dst_dsq = scx_resolve_local_dsq(sch, dst_rq, p, &enq_flags); - /* @p is on @dst_rq, an rq-owned @src_dsq is covered by the rq lock */ - if (!dsq_is_rq_owned(src_dsq)) - lockdep_assert_held(&src_dsq->lock); lockdep_assert_rq_held(dst_rq); WARN_ON_ONCE(p->scx.holding_cpu >= 0); @@ -2629,8 +2642,8 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch, /* @p is going from a non-local DSQ to a local DSQ */ if (src_rq == dst_rq) { scx_task_unlink_from_dsq(p, src_dsq); - scx_move_local_task_to_local_dsq(sch, p, enq_flags, src_dsq, dst_rq); raw_spin_unlock(&src_dsq->lock); + scx_move_local_task_to_local_dsq(sch, p, enq_flags, dst_rq); } else { raw_spin_unlock(&src_dsq->lock); move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq); @@ -2680,8 +2693,8 @@ bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq, if (rq == task_rq) { scx_task_unlink_from_dsq(p, dsq); - scx_move_local_task_to_local_dsq(sch, p, enq_flags, dsq, rq); raw_spin_unlock(&dsq->lock); + scx_move_local_task_to_local_dsq(sch, p, enq_flags, rq); return true; } @@ -3629,8 +3642,12 @@ static void set_cpus_allowed_scx(struct task_struct *p, * * Fine-grained memory write control is enforced by BPF making the const * designation pointless. Cast it away when calling the operation. + * + * The cid form receives the initial mask when the task is enabled and + * hears about changes only afterwards, see struct scx_enable_args. */ - if (SCX_HAS_OP(sch, set_cpumask)) + if (SCX_HAS_OP(sch, set_cpumask) && + (!scx_is_cid_type() || scx_get_task_state(p) == SCX_TASK_ENABLED)) scx_call_op_set_cpumask(sch, task_rq(p), p, (struct cpumask *)p->cpus_ptr); } @@ -3939,8 +3956,27 @@ static void __scx_enable_task(struct scx_sched *sch, struct task_struct *p) p->scx.weight = sched_weight_to_cgroup(weight); - if (SCX_HAS_OP(sch, enable)) - SCX_CALL_OP_TASK(sch, enable, rq, p); + if (SCX_HAS_OP(sch, enable)) { + if (scx_is_cid_type()) { + struct scx_cmask *cmask = scx_fill_cmask_scratch(sch, p->cpus_ptr); + struct scx_enable_args args = { + .cmask_arena_addr = scx_kaddr_to_arena(sch, cmask), + }; + + SCX_CALL_CID_OP_TASK(sch, enable, rq, p, &args); + } else { + SCX_CALL_OP_TASK(sch, enable, rq, p); + } + } + + /* + * The initial mask also goes out through set_cmask() so a scheduler can + * track affinity there alone, and before set_weight() so that the mask + * is in place when weight-dependent state is derived, see struct + * scx_enable_args. + */ + if (scx_is_cid_type() && SCX_HAS_OP(sch, set_cpumask)) + scx_call_op_set_cpumask(sch, rq, p, p->cpus_ptr); if (SCX_HAS_OP(sch, set_weight)) SCX_CALL_OP_TASK(sch, set_weight, rq, p, p->scx.weight); @@ -4283,9 +4319,10 @@ static void switching_to_scx(struct rq *rq, struct task_struct *p) /* * set_cpus_allowed_scx() is not called while @p is associated with a - * different scheduler class. Keep the BPF scheduler up-to-date. + * different scheduler class. Keep the BPF scheduler up-to-date. The cid + * form gets its mask from scx_enable_task(). */ - if (SCX_HAS_OP(sch, set_cpumask)) + if (!scx_is_cid_type() && SCX_HAS_OP(sch, set_cpumask)) scx_call_op_set_cpumask(sch, rq, p, (struct cpumask *)p->cpus_ptr); } @@ -4404,6 +4441,17 @@ static bool local_task_should_reenq(struct rq *rq, struct task_struct *p, return *reenq_flags & SCX_REENQ_ANY; } +/* + * The dispatcher stores the final ops_state after dropping the DSQ lock, so @p + * can be found on a DSQ while still %SCX_OPSS_DISPATCHING. Reenqueueing @p + * before that store lands would have it clobber the new %SCX_OPSS_QUEUED. + */ +void scx_reenq_wait_dispatching(struct task_struct *p) +{ + if (unlikely(atomic_long_read_acquire(&p->scx.ops_state) == SCX_OPSS_DISPATCHING)) + wait_ops_state(p, SCX_OPSS_DISPATCHING); +} + static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) { LIST_HEAD(tasks); @@ -4447,6 +4495,7 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) if (!local_task_should_reenq(rq, p, &reenq_flags, &reason)) continue; + scx_reenq_wait_dispatching(p); scx_dispatch_dequeue(rq, p); if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) @@ -4570,6 +4619,7 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag } /* @p is on @dsq, its rq and @dsq are locked */ + scx_reenq_wait_dispatching(p); dispatch_dequeue_locked(p, dsq); raw_spin_unlock(&dsq->lock); @@ -8356,10 +8406,11 @@ static struct bpf_struct_ops bpf_sched_ext_ops = { /* * cid-form cfi stubs. Stubs whose signatures match the cpu-form (param types * identical, only param names differ across structs) are reused. Some need - * fresh stubs, set_cmask due to an argument type difference and the sub-sched - * notifiers because no cpu-form stub exists to reuse. + * fresh stubs, set_cmask and enable due to argument differences and the + * sub-sched notifiers because no cpu-form stub exists to reuse. */ static void sched_ext_ops_cid__set_cmask(struct task_struct *p, const struct scx_cmask *cmask__arena) {} +static void sched_ext_ops_cid__enable(struct task_struct *p, struct scx_enable_args *args) {} static void sched_ext_ops__sub_caps_updated(const struct scx_cmask *cmask__arena, u64 caps) {} static void sched_ext_ops__sub_ecaps_updated(s32 cid, u64 before, u64 after) {} @@ -8380,7 +8431,7 @@ static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { .update_idle = sched_ext_ops__update_idle, .init_task = sched_ext_ops__init_task, .exit_task = sched_ext_ops__exit_task, - .enable = sched_ext_ops__enable, + .enable = sched_ext_ops_cid__enable, .disable = sched_ext_ops__disable, #ifdef CONFIG_EXT_GROUP_SCHED .cpuctl_init = sched_ext_ops__cgroup_init, @@ -10403,8 +10454,7 @@ __bpf_kfunc const void *scx_bpf_online_cmask(const struct bpf_prog_aux *aux) if (unlikely(!online)) return NULL; - /* BPF rebases by the low 32 bits, like __arena callback args */ - return (void *)((unsigned long)online - sch->arena_kern_base); + return (void *)scx_kaddr_to_arena(sch, online); } /** diff --git a/kernel/sched/ext/inlines.h b/kernel/sched/ext/inlines.h index ed423bcc26b8..2ff5479334cb 100644 --- a/kernel/sched/ext/inlines.h +++ b/kernel/sched/ext/inlines.h @@ -129,8 +129,10 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, * scheduler's ops.dispatch() doesn't yield any tasks. */ if (scx_bypass_dsp_enabled(sch) && - scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) + scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) { + __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); return SCX_DSP_LOCAL; + } return SCX_DSP_NONE; } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 3464e0f113c1..1df8f583b0ec 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -250,6 +250,31 @@ struct scx_exit_task_args { bool cancelled; }; +/** + * struct scx_enable_args - Argument container for cid-form ops.enable() + * @cmask_arena_addr: BPF arena address of the cmask of cids the task may run on + * + * @cmask_arena_addr is the task's affinity as it enters the scheduler. + * set_cmask() delivers the same mask right after enable(), before set_weight() + * and the first enqueue, then every affinity change afterwards, and is never + * called before enable(). A scheduler may therefore track affinity in + * set_cmask() alone. + * + * The kernel builds the mask in the scheduler arena from its own geometry, so + * the header is valid regardless of what the scheduler last wrote there. The + * memory is per-cpu scratch reused once the callback returns: copy the bits + * out, don't keep the address. The set_cmask() argument follows the same rules. + * + * The address is a plain value rather than a typed pointer because BTF can't + * mark a struct member as an arena pointer yet and a pointer member would reach + * the program typed as a kernel pointer. Cast it to struct scx_cmask __arena * + * before use. Once arena members can be typed, a typed alias will join this + * field in an anonymous union at the same offset. + */ +struct scx_enable_args { + u64 cmask_arena_addr; +}; + /* argument container for ops.cgroup_init() */ struct scx_cgroup_init_args { /* the weight of the cgroup [1..10000] */ @@ -1037,6 +1062,7 @@ struct sched_ext_ops { * - dispatch -> dispatch (cpu arg is now cid) * - update_idle -> update_idle (cpu arg is now cid) * - set_cpumask -> set_cmask (cmask instead of cpumask) + * - enable -> enable (takes struct scx_enable_args) * - cpu_online -> cid_online * - cpu_offline -> cid_offline * - dump_cpu -> dump_cid @@ -1070,7 +1096,7 @@ struct sched_ext_ops_cid { struct scx_init_task_args *args); void (*exit_task)(struct task_struct *p, struct scx_exit_task_args *args); - void (*enable)(struct task_struct *p); + void (*enable)(struct task_struct *p, struct scx_enable_args *args); void (*disable)(struct task_struct *p); void (*dump)(struct scx_dump_ctx *ctx); void (*dump_cid)(struct scx_dump_ctx *ctx, s32 cid, bool idle); @@ -1533,7 +1559,8 @@ struct scx_sched { * by BUILD_BUG_ON in scx_init()). The anonymous union lets the kernel * access either view of the same storage without function-pointer * casts: use .ops for cpu-form and shared fields, .ops_cid for the - * cid-renamed callbacks (set_cmask, select_cid, cid_online, ...). + * callbacks whose cid-form signature differs (set_cmask, enable, + * select_cid, cid_online, ...). */ union { struct sched_ext_ops ops; @@ -1556,9 +1583,9 @@ struct scx_sched { uintptr_t arena_kern_base; /* - * Per-CPU arena cmask used by scx_call_op_set_cpumask() to hand a cmask - * to ops_cid.set_cmask(). The kernel writes through the stored kern_va - * and passes it to the callback's __arena argument. + * Per-CPU arena cmask the kernel fills from a task's cpumask and hands + * to ops_cid.enable() and ops_cid.set_cmask(). The stored pointers are + * the kernel addresses. */ struct scx_cmask * __percpu *set_cmask_scratch; struct scx_cmask *online_cmask; @@ -1669,6 +1696,19 @@ static inline void *scx_arena_to_kaddr(struct scx_sched *sch, const void *bpf_pt return (void *)(sch->arena_kern_base + (u32)(uintptr_t)bpf_ptr); } +/** + * scx_kaddr_to_arena - Translate a kernel arena address to the BPF form + * @sch: scheduler whose arena hosts @kaddr + * @kaddr: kernel address inside @sch's arena + * + * __arena callback arguments need no translation. Addresses handed to BPF any + * other way, such as struct fields and kfunc return values, go through this. + */ +static inline uintptr_t scx_kaddr_to_arena(struct scx_sched *sch, const void *kaddr) +{ + return (uintptr_t)kaddr - sch->arena_kern_base; +} + enum scx_wake_flags { /* expose select WF_* flags as enums */ SCX_WAKE_FORK = WF_FORK, @@ -2065,8 +2105,7 @@ void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p); void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, int sticky_cpu); void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct *p, - u64 enq_flags, struct scx_dispatch_q *src_dsq, - struct rq *dst_rq); + u64 enq_flags, struct rq *dst_rq); bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, u64 enq_flags); bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq); @@ -2078,6 +2117,7 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags); u64 __scx_bpf_now(struct rq *rq); void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, u64 reenq_flags, struct rq *locked_rq); +void scx_reenq_wait_dispatching(struct task_struct *p); int __scx_init_task(struct scx_sched *sch, struct task_struct *p, struct cgroup *cgrp, bool fork); void scx_enable_task(struct scx_sched *sch, struct task_struct *p); @@ -2302,9 +2342,9 @@ do { \ } while (0) /* - * Dispatch a task op through the cid-form ops_cid table. Only set_cmask() needs - * this: it takes an arena cmask address instead of a cpumask, so it cannot be - * invoked via its cpu-form set_cpumask() slot. + * Dispatch a task op through the cid-form ops_cid table, for the ops whose + * cid-form signature differs from the cpu-form slot: set_cmask() takes an arena + * cmask instead of a cpumask and enable() takes scx_enable_args. */ #define SCX_CALL_CID_OP_TASK(sch, op, locked_rq, task, args...) \ __SCX_CALL_OP_TASK(sch, ops_cid, op, locked_rq, task, ##args) diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 34e642a1a403..0472eaf41c7c 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -555,8 +555,8 @@ static void scx_rescue_timerfn(struct timer_list *timer) scx.dsq_list.node); scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq); scx_rescue_admit(rq, p, slice); - scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_IGNORE_CAPS, - &rq->scx.rescue.dsq, rq); + scx_move_local_task_to_local_dsq(scx_task_sched(p), p, + SCX_ENQ_IGNORE_CAPS, rq); if (sched_class_above(&ext_sched_class, rq->curr->sched_class)) resched_curr(rq); } else if (p->scx.dsq && rq->scx.rescue.budget > 2 * scx_rescue_quantum_ns) { @@ -572,7 +572,7 @@ static void scx_rescue_timerfn(struct timer_list *timer) scx_task_unlink_from_dsq(p, &rq->scx.local_dsq); scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_HEAD | SCX_ENQ_PREEMPT | SCX_ENQ_IGNORE_CAPS, - &rq->scx.local_dsq, rq); + rq); } out_arm: scx_rescue_timer_arm(rq); @@ -596,8 +596,8 @@ void scx_rescue_flush(struct rq *rq) /* and flush out all pending ones */ list_for_each_entry_safe(p, n, &rq->scx.rescue.dsq.list, scx.dsq_list.node) { scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq); - scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_IGNORE_CAPS, - &rq->scx.rescue.dsq, rq); + scx_move_local_task_to_local_dsq(scx_task_sched(p), p, + SCX_ENQ_IGNORE_CAPS, rq); } timer_delete(&rq->scx.rescue.timer); @@ -801,6 +801,7 @@ void scx_reenq_reject(struct rq *rq) if (WARN_ON_ONCE(p->migration_pending)) continue; + scx_reenq_wait_dispatching(p); scx_dispatch_dequeue(rq, p); if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) diff --git a/kernel/sched/ext/types.h b/kernel/sched/ext/types.h index 943d8d429a2c..139176cf9fc6 100644 --- a/kernel/sched/ext/types.h +++ b/kernel/sched/ext/types.h @@ -70,6 +70,10 @@ enum scx_consts { * smaller shards if the LLC exceeds the target size. No-topo cids are packed * into their own max-sized shards. * + * New fields are appended, never inserted: scx_bpf_cid_topo() copies this + * struct out sized by the program's own layout, and an older program's copy + * must stay a prefix of the kernel's. + * * @core_cid: first cid of this cid's core (smt-sibling group) * @core_idx: global index of that core, in [0, nr_cores_at_init) * @llc_cid: first cid of this cid's LLC diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 7455a83a6a99..57360f5cdde4 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1478,7 +1478,7 @@ static inline int get_sched_cache_scale(int mul) return (1 + (tol - 1) * mul); } -static bool exceed_llc_capacity(struct mm_struct *mm, int cpu) +static bool exceed_llc_capacity(struct sched_cache_group *grp, int cpu) { #ifdef CONFIG_NUMA_BALANCING unsigned long llc, footprint; @@ -1497,7 +1497,7 @@ static bool exceed_llc_capacity(struct mm_struct *mm, int cpu) * excluded. */ llc = sd->llc_bytes; - footprint = READ_ONCE(mm->sc_stat.footprint); + footprint = READ_ONCE(grp->footprint); /* * Scale the LLC size by 256*llc_aggr_tolerance @@ -1526,7 +1526,7 @@ static bool exceed_llc_capacity(struct mm_struct *mm, int cpu) return false; } -static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p, +static bool invalid_llc_nr(struct sched_cache_group *grp, struct task_struct *p, int cpu) { int scale; @@ -1542,10 +1542,32 @@ static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p, if (scale == INT_MAX) return false; - return !fits_capacity((mm->sc_stat.nr_running_avg * cpu_smt_num_threads), + return !fits_capacity((READ_ONCE(grp->nr_running_avg) * cpu_smt_num_threads), (scale * per_cpu(sd_llc_size, cpu))); } +/* + * A task counts in nr_pref_llc_running while it is queued on its preferred + * LLC (pref_llc_queued) and runnable (!sched_delayed), keeping the counter in + * the runnable domain so alb_break_llc() can compare it with h_nr_runnable. + */ +static bool task_pref_llc_runnable(struct task_struct *p) +{ + return p->pref_llc_queued && !p->se.sched_delayed; +} + +static void pref_llc_running_inc(struct rq *rq, struct task_struct *p) +{ + if (task_pref_llc_runnable(p)) + rq->nr_pref_llc_running++; +} + +static void pref_llc_running_dec(struct rq *rq, struct task_struct *p) +{ + if (task_pref_llc_runnable(p)) + rq->nr_pref_llc_running--; +} + static void account_llc_enqueue(struct rq *rq, struct task_struct *p) { int pref_llc, pref_llc_queued; @@ -1557,7 +1579,6 @@ static void account_llc_enqueue(struct rq *rq, struct task_struct *p) pref_llc_queued = (pref_llc == task_llc(p)); rq->nr_llc_running++; - rq->nr_pref_llc_running += pref_llc_queued; /* * Record whether p is enqueued on its preferred @@ -1575,6 +1596,9 @@ static void account_llc_enqueue(struct rq *rq, struct task_struct *p) */ p->pref_llc_queued = pref_llc_queued; + /* Skipped while delayed; clear_delayed() adds it back on wake. */ + pref_llc_running_inc(rq, p); + sd = rcu_dereference_all(rq->sd); if (sd && (unsigned int)pref_llc < sd->llc_max) sd->llc_counts[pref_llc]++; @@ -1591,7 +1615,12 @@ static void account_llc_dequeue(struct rq *rq, struct task_struct *p) rq->nr_llc_running--; if (p->pref_llc_queued) { - rq->nr_pref_llc_running--; + /* + * Skipped if still delayed (set_delayed() already removed it); + * clearing pref_llc_queued below also stops clear_delayed() + * from re-adding it. + */ + pref_llc_running_dec(rq, p); /* * Update the status in case * other logic might query @@ -1619,12 +1648,20 @@ static void account_llc_dequeue(struct rq *rq, struct task_struct *p) } } -void mm_init_sched(struct mm_struct *mm, - struct sched_cache_time __percpu *_pcpu_sched) +int mm_init_sched(struct mm_struct *mm, + struct sched_cache_time __percpu *_pcpu_sched) { + struct sched_cache_group *grp; unsigned long epoch = 0; int i; + grp = kzalloc_obj(*grp); + if (!grp) { + free_percpu(_pcpu_sched); + mm->sched_cache_grp = NULL; + return -ENOMEM; + } + for_each_possible_cpu(i) { struct sched_cache_time *pcpu_sched = per_cpu_ptr(_pcpu_sched, i); struct rq *rq = cpu_rq(i); @@ -1635,18 +1672,141 @@ void mm_init_sched(struct mm_struct *mm, epoch = rq->cpu_epoch; } - raw_spin_lock_init(&mm->sc_stat.lock); - mm->sc_stat.epoch = epoch; - mm->sc_stat.cpu = -1; - mm->sc_stat.next_scan = jiffies; - mm->sc_stat.nr_running_avg = 0; - mm->sc_stat.footprint = 0; + raw_spin_lock_init(&grp->lock); + grp->epoch = epoch; + grp->cpu = -1; + grp->next_scan = jiffies; + grp->nr_running_avg = 0; + grp->footprint = 0; + refcount_set(&grp->refcnt, 1); /* - * The update to mm->sc_stat should not be reordered - * before initialization to mm's other fields, in case + * The update to grp->pcpu_sched should not be reordered + * before initialization to grp's other fields, in case * the readers may get invalid mm_sched_epoch, etc. */ - smp_store_release(&mm->sc_stat.pcpu_sched, _pcpu_sched); + smp_store_release(&grp->pcpu_sched, _pcpu_sched); + /* + * Publish the group last. Not every reader qualifies it by + * grp->pcpu_sched - can_migrate_llc_task() only checks that the + * pointer is non-NULL before reading grp->footprint and + * grp->nr_running_avg - so a reachable group must already be + * fully initialized. + */ + smp_store_release(&mm->sched_cache_grp, grp); + return 0; +} + +static void sched_cache_group_free_rcu(struct rcu_head *rcu) +{ + struct sched_cache_group *grp = + container_of(rcu, struct sched_cache_group, rcu); + + free_percpu(grp->pcpu_sched); + kfree(grp); +} + +static void sched_cache_group_put(struct sched_cache_group *grp) +{ + if (!grp || !refcount_dec_and_test(&grp->refcnt)) + return; + + call_rcu(&grp->rcu, sched_cache_group_free_rcu); +} + +DEFINE_FREE(sched_cache_group_put, struct sched_cache_group *, + sched_cache_group_put(_T)); + +#define rcu_deref_sched_cache_grp(tsk) \ + rcu_dereference_check((tsk)->sched_cache_grp, (tsk) == current) + +static struct sched_cache_group *sched_cache_replace_grp(struct task_struct *p, + struct sched_cache_group *new) +{ + struct sched_cache_group *old; + + old = rcu_deref_sched_cache_grp(p); + rcu_assign_pointer(p->sched_cache_grp, new); + + return old; +} + +struct sched_cache_group *sched_cache_group_get(struct sched_cache_group *grp) +{ + /* + * refcount_inc_not_zero() is the acquire primitive for lockless + * (RCU) lookups; plain refcount_inc() would scribble the count if + * it already reached zero. Return NULL in that case. + */ + if (grp && !refcount_inc_not_zero(&grp->refcnt)) + grp = NULL; + + return grp; +} + +struct sched_cache_group *task_cache_group_get(struct task_struct *p) +{ + guard(rcu)(); + return sched_cache_group_get(rcu_dereference(p->sched_cache_grp)); +} + +void sched_cache_fork(struct task_struct *p) +{ + /* + * The child takes its own reference on the mm's cache group, separate + * from the reference held by the mm. @p is not yet visible to readers, + * so a plain initializing store is enough. + */ + RCU_INIT_POINTER(p->sched_cache_grp, + sched_cache_group_get(p->mm->sched_cache_grp)); +} + +void sched_cache_fork_cleanup(struct task_struct *p) +{ + /* + * A fork that fails after sched_cache_fork() never reaches exit_mm(), + * so drop the reference here. @p never became visible, so there are no + * concurrent readers and the reference we hold keeps the group alive. + */ + sched_cache_group_put(rcu_access_pointer(p->sched_cache_grp)); + RCU_INIT_POINTER(p->sched_cache_grp, NULL); +} + +void sched_cache_exec_mmap(struct task_struct *p, struct mm_struct *mm) +{ + struct sched_cache_group *old; + + /* + * Acquire the new reference before publishing the pointer, then drop + * the old one. @p is current and the only writer of its own pointer. + */ + old = sched_cache_replace_grp(p, sched_cache_group_get(mm->sched_cache_grp)); + sched_cache_group_put(old); +} + +void sched_cache_exit_mm(struct task_struct *p) +{ + struct sched_cache_group *grp = sched_cache_replace_grp(p, NULL); + +#ifdef CONFIG_NUMA_BALANCING + /* + * Subtract this task's footprint from the group before dropping the + * reference, so the group footprint converges as its threads exit. + * Unlocked for performance; clamp to avoid underflow. + */ + if (grp && p->total_numa_faults) { + unsigned long fp = READ_ONCE(grp->footprint); + unsigned long sub = min(fp, p->total_numa_faults); + + WRITE_ONCE(grp->footprint, fp - sub); + } +#endif + sched_cache_group_put(grp); +} + +void mm_destroy_sched(struct mm_struct *mm) +{ + sched_cache_group_put(mm->sched_cache_grp); + mm->sched_cache_grp = NULL; } /* because why would C be fully specified */ @@ -1697,14 +1857,14 @@ static unsigned long fraction_mm_sched(struct rq *rq, return div64_u64(NICE_0_LOAD * pcpu_sched->runtime, rq->cpu_runtime + 1); } -static int get_pref_llc(struct task_struct *p, struct mm_struct *mm) +static int get_pref_llc(struct task_struct *p, struct sched_cache_group *grp) { int mm_sched_llc = -1, mm_sched_cpu; - if (!mm) + if (!grp) return -1; - mm_sched_cpu = READ_ONCE(mm->sc_stat.cpu); + mm_sched_cpu = READ_ONCE(grp->cpu); if (mm_sched_cpu != -1) { mm_sched_llc = llc_id(mm_sched_cpu); @@ -1734,8 +1894,8 @@ static unsigned int task_running_on_cpu(int cpu, struct task_struct *p); static inline void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) { + struct sched_cache_group *grp = rcu_dereference_all(p->sched_cache_grp); struct sched_cache_time *pcpu_sched; - struct mm_struct *mm = p->mm; int mm_sched_llc = -1; unsigned long epoch; @@ -1746,12 +1906,18 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) return; /* * init_task, kthreads and user thread created - * by user_mode_thread() don't have mm. + * by user_mode_thread() don't have a cache group. + * In theory a kernel thread does not have any valid + * cache group, because sched_cache_fork() is not + * invoked for a kernel thread - !grp should gate the + * kernel thread. Use the PF_KTHREAD check explicitly + * here for safety reasons, to guard against future + * modifications and to pair with task_tick_cache(). */ - if (!mm || !mm->sc_stat.pcpu_sched) + if (p->flags & PF_KTHREAD || !grp || !grp->pcpu_sched) return; - pcpu_sched = per_cpu_ptr(mm->sc_stat.pcpu_sched, cpu_of(rq)); + pcpu_sched = per_cpu_ptr(grp->pcpu_sched, cpu_of(rq)); scoped_guard (raw_spinlock, &rq->cpu_epoch_lock) { __update_mm_sched(rq, pcpu_sched); @@ -1764,14 +1930,14 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) * If this process hasn't hit task_cache_work() for a while invalidate * its preferred state. */ - if ((long)(epoch - READ_ONCE(mm->sc_stat.epoch)) > llc_epoch_affinity_timeout || - invalid_llc_nr(mm, p, cpu_of(rq)) || - exceed_llc_capacity(mm, cpu_of(rq))) { - if (READ_ONCE(mm->sc_stat.cpu) != -1) - WRITE_ONCE(mm->sc_stat.cpu, -1); + if ((long)(epoch - READ_ONCE(grp->epoch)) > llc_epoch_affinity_timeout || + invalid_llc_nr(grp, p, cpu_of(rq)) || + exceed_llc_capacity(grp, cpu_of(rq))) { + if (READ_ONCE(grp->cpu) != -1) + WRITE_ONCE(grp->cpu, -1); } - mm_sched_llc = get_pref_llc(p, mm); + mm_sched_llc = get_pref_llc(p, grp); /* task not on rq accounted later in account_entity_enqueue() */ if (task_running_on_cpu(rq->cpu, p) && @@ -1784,31 +1950,32 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) static void task_tick_cache(struct rq *rq, struct task_struct *p) { + struct sched_cache_group *grp = rcu_dereference_all(p->sched_cache_grp); struct callback_head *work = &p->cache_work; - struct mm_struct *mm = p->mm; unsigned long epoch; if (!sched_cache_enabled()) return; - if (!mm || p->flags & PF_KTHREAD || - !mm->sc_stat.pcpu_sched) + if (!grp || p->flags & PF_KTHREAD || + !grp->pcpu_sched) return; epoch = rq->cpu_epoch; /* avoid moving backwards */ - if (time_after_eq(mm->sc_stat.epoch, epoch)) + if (time_after_eq(grp->epoch, epoch)) return; - guard(raw_spinlock)(&mm->sc_stat.lock); + guard(raw_spinlock)(&grp->lock); if (work->next == work) { task_work_add(p, work, TWA_RESUME); - WRITE_ONCE(mm->sc_stat.epoch, epoch); + WRITE_ONCE(grp->epoch, epoch); } } -static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p) +static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p, + struct sched_cache_group *grp) { #ifdef CONFIG_NUMA_BALANCING int cpu, curr_cpu, nid, pref_nid; @@ -1816,7 +1983,7 @@ static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p) if (!static_branch_likely(&sched_numa_balancing)) goto out; - cpu = READ_ONCE(p->mm->sc_stat.cpu); + cpu = READ_ONCE(grp->cpu); if (cpu != -1) nid = cpu_to_node(cpu); curr_cpu = task_cpu(p); @@ -1873,13 +2040,13 @@ static inline void update_avg_scale(u64 *avg, u64 sample) static void task_cache_work(struct callback_head *work) { + struct sched_cache_group *grp __free(sched_cache_group_put) = NULL; + cpumask_var_t cpus __free(free_cpumask_var) = CPUMASK_VAR_NULL; int cpu, m_a_cpu = -1, nr_running = 0, curr_cpu; unsigned long next_scan, now = jiffies; struct task_struct *p = current, *cur; unsigned long curr_m_a_occ = 0; - struct mm_struct *mm = p->mm; unsigned long m_a_occ = 0; - cpumask_var_t cpus; WARN_ON_ONCE(work != &p->cache_work); @@ -1888,21 +2055,30 @@ static void task_cache_work(struct callback_head *work) if (p->flags & PF_EXITING) return; - next_scan = READ_ONCE(mm->sc_stat.next_scan); + /* + * A reference makes sure grp is not released by others. The rcu + * lock can not be held till after zalloc_cpumask_var() below, + * because the latter might sleep. + */ + grp = task_cache_group_get(p); + if (!grp) + return; + + next_scan = READ_ONCE(grp->next_scan); if (time_before(now, next_scan)) return; /* only 1 thread is allowed to scan */ - if (!try_cmpxchg(&mm->sc_stat.next_scan, &next_scan, + if (!try_cmpxchg(&grp->next_scan, &next_scan, now + max_t(unsigned long, READ_ONCE(llc_epoch_period), 1))) return; curr_cpu = task_cpu(p); - if (invalid_llc_nr(mm, p, curr_cpu) || - exceed_llc_capacity(mm, curr_cpu)) { - if (READ_ONCE(mm->sc_stat.cpu) != -1) - WRITE_ONCE(mm->sc_stat.cpu, -1); + if (invalid_llc_nr(grp, p, curr_cpu) || + exceed_llc_capacity(grp, curr_cpu)) { + if (READ_ONCE(grp->cpu) != -1) + WRITE_ONCE(grp->cpu, -1); return; } @@ -1913,7 +2089,7 @@ static void task_cache_work(struct callback_head *work) scoped_guard (cpus_read_lock) { guard(rcu)(); - get_scan_cpumasks(cpus, p); + get_scan_cpumasks(cpus, p, grp); for_each_cpu(cpu, cpus) { /* XXX sched_cluster_active */ @@ -1926,16 +2102,20 @@ static void task_cache_work(struct callback_head *work) for_each_cpu(i, sched_domain_span(sd)) { occ = fraction_mm_sched(cpu_rq(i), - per_cpu_ptr(mm->sc_stat.pcpu_sched, i)); + per_cpu_ptr(grp->pcpu_sched, i)); a_occ += occ; if (occ > m_occ) { m_occ = occ; m_cpu = i; } + /* + * rcu_access_pointer() is used because the + * pointer is only compared, never dereferenced. + */ cur = rcu_dereference_all(cpu_rq(i)->curr); if (cur && !(cur->flags & (PF_EXITING | PF_KTHREAD)) && - cur->mm == mm) + rcu_access_pointer(cur->sched_cache_grp) == grp) nr_running++; } @@ -1959,7 +2139,7 @@ static void task_cache_work(struct callback_head *work) m_a_cpu = m_cpu; } - if (llc_id(cpu) == llc_id(READ_ONCE(mm->sc_stat.cpu))) + if (llc_id(cpu) == llc_id(READ_ONCE(grp->cpu))) curr_m_a_occ = a_occ; cpumask_andnot(cpus, cpus, sched_domain_span(sd)); @@ -1968,7 +2148,7 @@ static void task_cache_work(struct callback_head *work) if (m_a_occ > (2 * curr_m_a_occ)) { /* - * Avoid switching sc_stat.cpu too fast. + * Avoid switching sched_cache_grp->cpu too fast. * The reason to choose 2X is because: * 1. It is better to keep the preferred LLC stable, * rather than changing it frequently and cause migrations @@ -1977,11 +2157,10 @@ static void task_cache_work(struct callback_head *work) * 3. 2X is chosen based on test results, as it delivers * the optimal performance gain so far. */ - WRITE_ONCE(mm->sc_stat.cpu, m_a_cpu); + WRITE_ONCE(grp->cpu, m_a_cpu); } - update_avg_scale(&mm->sc_stat.nr_running_avg, nr_running); - free_cpumask_var(cpus); + update_avg_scale(&grp->nr_running_avg, nr_running); } void init_sched_mm(struct task_struct *p) @@ -1990,11 +2169,19 @@ void init_sched_mm(struct task_struct *p) init_task_work(work, task_cache_work); work->next = work; + /* + * dup_task_struct() copies the parent's task_struct, including its + * sched_cache_grp, for which the child holds no reference. Clear it + * here - before copy_mm() runs - so the child never carries a + * borrowed pointer that the fork error path would put. + */ + RCU_INIT_POINTER(p->sched_cache_grp, NULL); /* * Reset new task's preference to avoid * polluting account_llc_enqueue(). */ p->preferred_llc = -1; + p->pref_llc_queued = 0; } #else /* CONFIG_SCHED_CACHE */ @@ -2016,6 +2203,10 @@ static void account_llc_enqueue(struct rq *rq, struct task_struct *p) {} static void account_llc_dequeue(struct rq *rq, struct task_struct *p) {} +static void pref_llc_running_inc(struct rq *rq, struct task_struct *p) {} + +static void pref_llc_running_dec(struct rq *rq, struct task_struct *p) {} + #endif /* CONFIG_SCHED_CACHE */ /* @@ -3692,6 +3883,7 @@ static int preferred_group_nid(struct task_struct *p, int nid) static void task_numa_placement(struct task_struct *p) __context_unsafe(/* conditional locking */) { + struct sched_cache_group __maybe_unused *grp; int seq, nid, max_nid = NUMA_NO_NODE; unsigned long max_faults = 0; unsigned long fault_types[2] = { 0, 0 }; @@ -3784,19 +3976,24 @@ static void task_numa_placement(struct task_struct *p) * heuristic and occasional lost updates are tolerable. * * If a task exits, its corresponding footprint must - * be subtracted from the mm->sc_stat.footprint, otherwise - * the mm->sc_stat.footprint will not converge: - * the exiting thread's footprint remains unchanged/undecayed - * in mm->sc_stat.footprint. See exit_mm(). + * be subtracted from p->sched_cache_grp->footprint, + * otherwise the footprint will not converge: the + * exiting thread's footprint remains unchanged/undecayed. + * See exit_mm(). * * Lost updates and unsynchronized subtraction * in exit_mm() can cause footprint + diff to * go negative. Clamp to zero to prevent the * unsigned footprint from wrapping. */ - new_fp = (long)READ_ONCE(p->mm->sc_stat.footprint) + diff; - WRITE_ONCE(p->mm->sc_stat.footprint, - max(new_fp, 0L)); + scoped_guard(rcu) { + grp = rcu_dereference(p->sched_cache_grp); + + if (grp) { + new_fp = (long)READ_ONCE(grp->footprint) + diff; + WRITE_ONCE(grp->footprint, max(new_fp, 0L)); + } + } #endif } @@ -6390,15 +6587,27 @@ static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq); static void set_delayed(struct sched_entity *se) { - se->sched_delayed = 1; - /* * Delayed se of cfs_rq have no tasks queued on them. * Do not adjust h_nr_runnable since __dequeue_task() * will account it for blocked tasks. + * + * This check can be removed because when flat pick + * patches get merged as only task can get delayed, + * same for clear_delayed(). */ - if (!entity_is_task(se)) + if (!entity_is_task(se)) { + se->sched_delayed = 1; return; + } + + /* + * Drop a task leaving the runnable set. + * Needs to be called before sched_delayed is set. + * clear_delayed() mirrors this after clearing the flag. + */ + pref_llc_running_dec(rq_of(cfs_rq_of(se)), task_of(se)); + se->sched_delayed = 1; for_each_sched_entity(se) { struct cfs_rq *cfs_rq = cfs_rq_of(se); @@ -6420,6 +6629,13 @@ static void clear_delayed(struct sched_entity *se) if (!entity_is_task(se)) return; + /* + * Re-add on wake, after sched_delayed is cleared. On a final delayed + * dequeue account_llc_dequeue() already cleared pref_llc_queued, so + * this does nothing. + */ + pref_llc_running_inc(rq_of(cfs_rq_of(se)), task_of(se)); + for_each_sched_entity(se) { struct cfs_rq *cfs_rq = cfs_rq_of(se); @@ -10395,6 +10611,7 @@ enum migration_type { #define LBF_SOME_PINNED 0x08 #define LBF_ACTIVE_LB 0x10 #define LBF_LLC_PINNED 0x20 +#define LBF_ACTIVE_LB_LLC 0x40 struct lb_env { struct sched_domain *sd; @@ -10724,7 +10941,7 @@ static inline bool task_misfits_asym_cpu(struct lb_env *env, struct task_struct static enum llc_mig can_migrate_llc_task(struct lb_env *env, struct task_struct *p) { - struct mm_struct *mm; + struct sched_cache_group *grp; bool to_pref; int cpu, src_cpu, dst_cpu; @@ -10733,19 +10950,19 @@ static enum llc_mig can_migrate_llc_task(struct lb_env *env, src_cpu = env->src_cpu; dst_cpu = env->dst_cpu; - mm = p->mm; - if (!mm) + grp = rcu_dereference_all(p->sched_cache_grp); + if (!grp) return mig_unrestricted; - cpu = READ_ONCE(mm->sc_stat.cpu); + cpu = READ_ONCE(grp->cpu); if (cpu < 0 || cpus_share_cache(src_cpu, dst_cpu)) return mig_unrestricted; /* skip cache aware load balance for too many threads */ - if (invalid_llc_nr(mm, p, dst_cpu) || - exceed_llc_capacity(mm, dst_cpu)) { - if (READ_ONCE(mm->sc_stat.cpu) != -1) - WRITE_ONCE(mm->sc_stat.cpu, -1); + if (invalid_llc_nr(grp, p, dst_cpu) || + exceed_llc_capacity(grp, dst_cpu)) { + if (READ_ONCE(grp->cpu) != -1) + WRITE_ONCE(grp->cpu, -1); return mig_unrestricted; } @@ -10813,6 +11030,21 @@ alb_break_llc(struct lb_env *env) return false; } +/* + * Returns true if p's preferred LLC does not match the destination CPU + * under migrate_llc_task semantics. Passive LB passes migrate_llc_task + * in env->migration_type, while active LB carries LBF_ACTIVE_LB_LLC in + * env->flags to avoid overwriting env->migration_type. + */ +static inline bool +migrate_llc_task_wrong_dst(struct task_struct *p, struct lb_env *env) +{ + return sched_cache_enabled() && + (env->migration_type == migrate_llc_task || + env->flags & LBF_ACTIVE_LB_LLC) && + READ_ONCE(p->preferred_llc) != llc_id(env->dst_cpu); +} + /* * Check if migrating task p from env->src_cpu to * env->dst_cpu breaks LLC localiy. @@ -10841,8 +11073,7 @@ static bool migrate_degrades_llc(struct task_struct *p, struct lb_env *env) * run on env->dst_cpu, skip the tasks do not prefer * env->dst_cpu, and find the one that prefers. */ - if (env->migration_type == migrate_llc_task && - READ_ONCE(p->preferred_llc) != llc_id(env->dst_cpu)) + if (migrate_llc_task_wrong_dst(p, env)) return true; if (can_migrate_llc_task(env, p) != mig_forbid) @@ -10864,6 +11095,12 @@ alb_break_llc(struct lb_env *env) return false; } +static inline bool +migrate_llc_task_wrong_dst(struct task_struct *p, struct lb_env *env) +{ + return false; +} + static inline bool migrate_degrades_llc(struct task_struct *p, struct lb_env *env) { @@ -10963,7 +11200,7 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) * 4) too many balance attempts have failed. */ if (env->flags & LBF_ACTIVE_LB) - return 1; + return !migrate_llc_task_wrong_dst(p, env); degrades = migrate_degrades_locality(p, env); if (!degrades) { @@ -13362,6 +13599,20 @@ static int need_active_balance(struct lb_env *env) } static int active_load_balance_cpu_stop(void *data); +static int active_load_balance_llc_cpu_stop(void *data); + +/* + * migration_type is checked elsewhere to decide migration policy, so + * it shouldn't be repurposed just to flag an LLC-directed active + * balance across the stopper. Pick the callback here instead. + */ +static inline cpu_stop_fn_t alb_stop_fn(struct lb_env *env) +{ + if (env->migration_type == migrate_llc_task) + return active_load_balance_llc_cpu_stop; + + return active_load_balance_cpu_stop; +} static int should_we_balance(struct lb_env *env) { @@ -13707,7 +13958,7 @@ static int sched_balance_rq(int this_cpu, struct rq *this_rq, } if (active_balance) { stop_one_cpu_nowait(cpu_of(busiest), - active_load_balance_cpu_stop, busiest, + alb_stop_fn(&env), busiest, &busiest->active_balance_work); } preempt_enable(); @@ -13812,7 +14063,7 @@ update_next_balance(struct sched_domain *sd, unsigned long *next_balance) * least 1 task to be running on each physical CPU where possible, and * avoids physical / logical imbalances. */ -static int active_load_balance_cpu_stop(void *data) +static int __active_load_balance_cpu_stop(void *data, unsigned int lb_flags) { struct rq *busiest_rq = data; int busiest_cpu = cpu_of(busiest_rq); @@ -13862,7 +14113,7 @@ static int active_load_balance_cpu_stop(void *data) .src_cpu = busiest_rq->cpu, .src_rq = busiest_rq, .idle = CPU_IDLE, - .flags = LBF_ACTIVE_LB, + .flags = LBF_ACTIVE_LB | lb_flags, }; schedstat_inc(sd->alb_count); @@ -13890,6 +14141,16 @@ static int active_load_balance_cpu_stop(void *data) return 0; } +static int active_load_balance_cpu_stop(void *data) +{ + return __active_load_balance_cpu_stop(data, 0); +} + +static int active_load_balance_llc_cpu_stop(void *data) +{ + return __active_load_balance_cpu_stop(data, LBF_ACTIVE_LB_LLC); +} + /* * Scale the max sched_balance_rq interval with the number of CPUs in the system. * This trades load-balance latency on larger machines for less cross talk. diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 0248227d983a..3dab0253976f 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -985,8 +985,8 @@ void sched_cache_active_set(void) } /* - * Update the bottom sched_domain's llc_bytes for @cpu and all its - * LLC siblings. Called from cacheinfo_cpu_online() or + * Update the bottom sched_domain's llc_bytes for @cpus sharing a physical + * LLC. Called from cacheinfo_cpu_online() or * cacheinfo_cpu_pre_down() with cpu hotplug lock held. * * Note: get_effective_llc_bytes() returns 0 on PowerPC. @@ -996,17 +996,13 @@ void sched_cache_active_set(void) * and does not populates the per-CPU struct cpu_cacheinfo array * that get_cpu_cacheinfo_llc() reads. */ -void sched_update_llc_bytes(unsigned int cpu) +void sched_update_llc_bytes(const struct cpumask *cpus) { struct sched_domain *sd, *sdp; unsigned int i; sched_domains_mutex_lock(); - sdp = rcu_dereference_sched_domain(per_cpu(sd_llc, cpu)); - if (!sdp) - goto unlock; - /* * ci->shared_cpu_map is built incrementally as CPUs come * online, so the first CPU in an LLC initially sees @@ -1014,14 +1010,22 @@ void sched_update_llc_bytes(unsigned int cpu) * get_effective_llc_bytes(). Re-evaluating every LLC * sibling on each online event corrects this once the full * shared_cpu_map is known. + * + * The departing CPU's domains have already been detached when + * cacheinfo removes it. Use the surviving cache siblings instead. + * They may belong to different cpuset partitions, so use each CPU's + * own LLC domain to scale its share of the physical cache. */ - for_each_cpu(i, sched_domain_span(sdp)) { + for_each_cpu(i, cpus) { + sdp = rcu_dereference_sched_domain(per_cpu(sd_llc, i)); + if (!sdp) + continue; + sd = rcu_dereference_sched_domain(cpu_rq(i)->sd); if (sd) sd->llc_bytes = get_effective_llc_bytes(i, sdp); } -unlock: sched_domains_mutex_unlock(); } diff --git a/kernel/sysctl.c b/kernel/sysctl.c index f7b75985d542..38597f26b34c 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -483,7 +483,7 @@ int proc_int_k2u_conv_kop(ulong *u_ptr, const int *k_ptr, bool *negp, if (val < 0) { *negp = true; - *u_ptr = k_ptr_op ? -k_ptr_op((ulong)val) : -(ulong)val; + *u_ptr = k_ptr_op ? k_ptr_op(-(ulong)val) : -(ulong)val; } else { *negp = false; *u_ptr = k_ptr_op ? k_ptr_op((ulong)val) : (ulong) val; diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c index 80c354811538..9b3487d40cd6 100644 --- a/kernel/time/jiffies.c +++ b/kernel/time/jiffies.c @@ -101,6 +101,8 @@ void __init register_refined_jiffies(long cycles_per_second) #ifdef CONFIG_SYSCTL static ulong mult_hz(const ulong val) { + if (val >= ULONG_MAX / HZ) + return ULONG_MAX; return val * HZ; } diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c index 1e9b00997ff2..da286619c5d8 100644 --- a/kernel/trace/fprobe.c +++ b/kernel/trace/fprobe.c @@ -47,6 +47,10 @@ static struct rhltable fprobe_ip_table; static DEFINE_MUTEX(fprobe_mutex); static struct fgraph_ops fprobe_graph_ops; +DEFINE_LOCK_GUARD_0(rcu_sched_notrace, + rcu_read_lock_sched_notrace(), + rcu_read_unlock_sched_notrace()) + static u32 fprobe_node_hashfn(const void *data, u32 len, u32 seed) { return hash_ptr(*(unsigned long **)data, 32); @@ -171,6 +175,11 @@ static inline bool write_fprobe_header(unsigned long *stack, static inline void read_fprobe_header(unsigned long *stack, struct fprobe **fp, unsigned int *size_words) { + if (!*stack) { + *fp = NULL; + *size_words = 0; + return; + } *fp = arch_decode_fprobe_header_fp(*stack); *size_words = arch_decode_fprobe_header_size(*stack); } @@ -203,6 +212,12 @@ static inline void read_fprobe_header(unsigned long *stack, { struct __fprobe_header *fph = (struct __fprobe_header *)stack; + if (!*stack) { + *fp = NULL; + *size_words = 0; + return; + } + *fp = fph->fp; *size_words = fph->size_words; } @@ -318,16 +333,14 @@ static void fprobe_ftrace_entry(unsigned long ip, unsigned long parent_ip, struct fprobe *fp; int bit; + if (!rcu_is_watching()) + return; + bit = ftrace_test_recursion_trylock(ip, parent_ip); if (bit < 0) return; - /* - * ftrace_test_recursion_trylock() disables preemption, but - * rhltable_lookup() checks whether rcu_read_lcok is held. - * So we take rcu_read_lock() here. - */ - rcu_read_lock(); + guard(rcu_sched_notrace)(); head = rhltable_lookup(&fprobe_ip_table, &ip, fprobe_rht_params); rhl_for_each_entry_rcu(node, pos, head, hlist) { @@ -342,7 +355,6 @@ static void fprobe_ftrace_entry(unsigned long ip, unsigned long parent_ip, else __fprobe_handler(ip, parent_ip, fp, fregs, NULL); } - rcu_read_unlock(); ftrace_test_recursion_unlock(bit); } NOKPROBE_SYMBOL(fprobe_ftrace_entry); @@ -556,10 +568,13 @@ static int fprobe_fgraph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops struct fprobe *fp; int used, ret; + if (!rcu_is_watching()) + return 0; + if (WARN_ON_ONCE(!fregs)) return 0; - guard(rcu)(); + guard(rcu_sched_notrace)(); head = rhltable_lookup(&fprobe_ip_table, &func, fprobe_rht_params); reserved_words = 0; rhl_for_each_entry_rcu(node, pos, head, hlist) { @@ -635,6 +650,10 @@ static int fprobe_fgraph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops } } + /* Terminate the list, fgraph_reserve_data() does not clear it. */ + if (used && used < reserved_words) + fgraph_data[used] = 0; + /* If any exit_handler is set, data must be used. */ return used != 0; } @@ -650,13 +669,16 @@ static void fprobe_return(struct ftrace_graph_ret *trace, int size, curr; int size_words; + if (!rcu_is_watching()) + return; + fgraph_data = (unsigned long *)fgraph_retrieve_data(gops->idx, &size); if (WARN_ON_ONCE(!fgraph_data)) return; size_words = SIZE_IN_LONG(size); ret_ip = ftrace_regs_get_instruction_pointer(fregs); - preempt_disable_notrace(); + guard(rcu_sched_notrace)(); curr = 0; while (size_words > curr) { @@ -672,7 +694,6 @@ static void fprobe_return(struct ftrace_graph_ret *trace, } curr += size; } - preempt_enable_notrace(); } NOKPROBE_SYMBOL(fprobe_return); diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 1ae3732a2c51..959525393739 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -3918,7 +3918,7 @@ static void check_flush_dependency(struct workqueue_struct *target_wq, WARN_ONCE(current->flags & PF_MEMALLOC, "workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%ps", current->pid, current->comm, target_wq->name, target_func); - WARN_ONCE(worker && ((worker->current_pwq->wq->flags & + WARN_ONCE(worker && worker->current_pwq && ((worker->current_pwq->wq->flags & (WQ_MEM_RECLAIM | __WQ_LEGACY)) == WQ_MEM_RECLAIM), "workqueue: WQ_MEM_RECLAIM %s:%ps is flushing !WQ_MEM_RECLAIM %s:%ps", worker->current_pwq->wq->name, worker->current_func, diff --git a/mm/backing-dev.c b/mm/backing-dev.c index cecbcf9060a6..18e999053bae 100644 --- a/mm/backing-dev.c +++ b/mm/backing-dev.c @@ -910,8 +910,9 @@ static void cleanup_offline_cgwbs_workfn(struct work_struct *work) continue; spin_unlock_irq(&cgwb_lock); - while (cleanup_offline_cgwb(wb)) - cond_resched(); + do { + cond_resched_tasks_rcu_qs(); + } while (cleanup_offline_cgwb(wb)); spin_lock_irq(&cgwb_lock); wb_put(wb); diff --git a/mm/damon/core.c b/mm/damon/core.c index 644daf5a1656..ab3c4d754964 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -2342,36 +2342,39 @@ static bool damos_skip_charged_region(struct damon_target *t, { struct damos_quota *quota = &s->quota; unsigned long sz_to_skip; + bool skip = false; /* Skip previously charged regions */ if (quota->charge_target_from) { if (t != quota->charge_target_from) return true; - if (r == damon_last_region(t)) { - quota->charge_target_from = NULL; - quota->charge_addr_from = 0; - return true; - } if (quota->charge_addr_from && - r->ar.end <= quota->charge_addr_from) - return true; + r->ar.end <= quota->charge_addr_from) { + skip = true; + goto out; + } if (quota->charge_addr_from && r->ar.start < quota->charge_addr_from) { sz_to_skip = ALIGN_DOWN(quota->charge_addr_from - r->ar.start, min_region_sz); if (!sz_to_skip) { - if (damon_sz_region(r) <= min_region_sz) - return true; + if (damon_sz_region(r) <= min_region_sz) { + skip = true; + goto out; + } sz_to_skip = min_region_sz; } damon_split_region_at(t, r, sz_to_skip); - return true; + skip = true; } + } +out: + if (r == damon_last_region(t)) { quota->charge_target_from = NULL; quota->charge_addr_from = 0; } - return false; + return skip; } static void damos_update_stat(struct damos *s, @@ -3091,6 +3094,7 @@ static void damos_set_effective_quota(struct damon_ctx *ctx, struct damos *s) struct damos_quota *quota = &s->quota; unsigned long throughput; unsigned long esz = ULONG_MAX; + unsigned long esz_time; if (!quota->ms && list_empty("a->goals)) { quota->esz = quota->sz; @@ -3111,8 +3115,8 @@ static void damos_set_effective_quota(struct damon_ctx *ctx, struct damos *s) 1000000, quota->total_charged_ns); else throughput = PAGE_SIZE * 1024; - esz = min(throughput * quota->ms, esz); - esz = max(ctx->min_region_sz, esz); + esz_time = max(throughput * quota->ms, ctx->min_region_sz); + esz = min(esz_time, esz); } if (quota->sz && quota->sz < esz) @@ -3143,7 +3147,7 @@ static void damos_adjust_quota(struct damon_ctx *c, struct damos *s) unsigned long cumulated_sz, cached_esz; unsigned int score, max_score = 0; - if (!quota->ms && !quota->sz && list_empty("a->goals)) + if (!damos_quota_is_set(quota)) return; /* First charge window */ @@ -3241,8 +3245,15 @@ static void kdamond_apply_schemes(struct damon_ctx *c) max_region_sz = damon_region_sz_limit(c); mutex_lock(&c->walk_control_lock); damon_for_each_target(t, c) { - if (c->ops.target_valid && c->ops.target_valid(t) == false) + if (c->ops.target_valid && c->ops.target_valid(t) == false) { + damon_for_each_scheme(s, c) { + if (s->quota.charge_target_from != t) + continue; + s->quota.charge_target_from = NULL; + s->quota.charge_addr_from = 0; + } continue; + } damos_apply_target(c, t, max_region_sz); } diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index fbda70d8ea4d..8fc61d06d358 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -61,7 +61,12 @@ void damon_ptep_mkold(pte_t *pte, struct vm_area_struct *vma, unsigned long addr * device aspects. */ if (likely(pte_present(pteval))) - young |= ptep_test_and_clear_young(vma, addr, pte); + /* + * Arch implementation of ptep_test_and_clear_young() may + * require aligned @addr + */ + young |= ptep_test_and_clear_young(vma, PAGE_ALIGN_DOWN(addr), + pte); young |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + PAGE_SIZE); if (young) folio_set_young(folio); diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 0648400b2d65..04ee2a2c6a4d 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -293,22 +293,29 @@ static int damon_mkold_pmd_entry(pmd_t *pmd, unsigned long addr, } #ifdef CONFIG_HUGETLB_PAGE +static bool damon_hugetlb_ptep_mkold(pte_t *pte, struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long addr, pte_t *entry) +{ + unsigned long psize = huge_page_size(hstate_vma(vma)); + + if (!pte_young(*entry)) + return false; + *entry = huge_ptep_get_and_clear(mm, addr, pte, psize); + *entry = pte_mkold(*entry); + set_huge_pte_at(mm, addr, pte, *entry, psize); + return true; +} + static void damon_hugetlb_mkold(pte_t *pte, struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr) { bool referenced = false; pte_t entry = huge_ptep_get(mm, addr, pte); struct folio *folio = pfn_folio(pte_pfn(entry)); - unsigned long psize = huge_page_size(hstate_vma(vma)); folio_get(folio); - if (pte_young(entry)) { - referenced = true; - entry = pte_mkold(entry); - set_huge_pte_at(mm, addr, pte, entry, psize); - } - + referenced = damon_hugetlb_ptep_mkold(pte, mm, vma, addr, &entry); if (mmu_notifier_clear_young(mm, addr, addr + huge_page_size(hstate_vma(vma)))) referenced = true; diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 4f6f58bf3db6..cea25773a6c9 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1967,6 +1967,15 @@ int dissolve_free_hugetlb_folio(struct folio *folio) struct hstate *h = folio_hstate(folio); bool adjust_surplus = false; + /* + * remove_hugetlb_folio()/update_and_free_hugetlb_folio() bail + * for gigantic hstates without runtime support, so dissolving one + * here would leave it on the free list and, on vmemmap restore + * failure, the add_hugetlb_folio() rollback corrupts that list. + */ + if (hstate_is_gigantic_no_runtime(h)) + goto out; + if (!available_huge_pages(h)) goto out; @@ -5161,18 +5170,21 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma, hugetlb_vma_lock_write(vma); i_mmap_lock_write(mapping); for (; old_addr < old_end; old_addr += sz, new_addr += sz) { + const unsigned long offset_to_last_entry = + (old_addr | last_addr_mask) - old_addr; + src_pte = hugetlb_walk(vma, old_addr, sz); if (!src_pte) { - old_addr |= last_addr_mask; - new_addr |= last_addr_mask; + old_addr += offset_to_last_entry; + new_addr += offset_to_last_entry; continue; } if (huge_pte_none(huge_ptep_get(mm, old_addr, src_pte))) continue; if (huge_pmd_unshare(&tlb, vma, old_addr, src_pte)) { - old_addr |= last_addr_mask; - new_addr |= last_addr_mask; + old_addr += offset_to_last_entry; + new_addr += offset_to_last_entry; continue; } diff --git a/mm/kasan/common.c b/mm/kasan/common.c index b7d05c2a6d93..7d21c4db68d2 100644 --- a/mm/kasan/common.c +++ b/mm/kasan/common.c @@ -114,12 +114,23 @@ void kasan_unpoison_task_stack(struct task_struct *task) /* Unpoison the stack for the current task beyond a watermark sp value. */ asmlinkage void kasan_unpoison_task_stack_below(const void *watermark) { + void *base; + + /* + * Only the generic mode needs this. In the tag-based mode the + * compiler fully initializes the shadow of stack variables on + * function entry, so stale tags left below the watermark are + * harmless. + */ + if (!IS_ENABLED(CONFIG_KASAN_GENERIC)) + return; + /* * Calculate the task stack base address. Avoid using 'current' * because this function is called by early resume code which hasn't * yet set up the percpu register (%gs). */ - void *base = (void *)((unsigned long)watermark & ~(THREAD_SIZE - 1)); + base = (void *)((unsigned long)watermark & ~(THREAD_SIZE - 1)); kasan_unpoison(base, watermark - base, false); } diff --git a/mm/memblock.c b/mm/memblock.c index 021db49eb7fc..e6fdf95a0534 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -2886,6 +2886,7 @@ static const char * const flagname[] = { [ilog2(MEMBLOCK_RSRV_NOINIT)] = "RSV_NIT", [ilog2(MEMBLOCK_RSRV_KERN)] = "RSV_KERN", [ilog2(MEMBLOCK_KHO_SCRATCH)] = "KHO_SCRATCH", + [ilog2(MEMBLOCK_RSRV_HUGETLB)] = "RSV_HUGETLB", }; static int memblock_debug_show(struct seq_file *m, void *private) diff --git a/mm/rmap.c b/mm/rmap.c index d1819fd69938..f3b21aaa34ee 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -209,7 +209,11 @@ int __anon_vma_prepare(struct vm_area_struct *vma) /* page_table_lock to protect against threads */ spin_lock(&mm->page_table_lock); if (likely(!vma->anon_vma)) { - vma->anon_vma = anon_vma; + /* + * Make anon_vma fields visible before anon_vma is published. + * Paired with an address dependency in reusable_anon_vma(). + */ + smp_store_release(&vma->anon_vma, anon_vma); anon_vma_chain_assign(vma, avc, anon_vma); anon_rmap_tree_insert(avc, anon_vma); anon_vma->num_active_vmas++; diff --git a/mm/shmem.c b/mm/shmem.c index 897fa2b61346..848316eaa7f4 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1847,8 +1847,11 @@ unsigned long shmem_allowable_huge_orders(struct inode *inode, global_orders = shmem_huge_global_enabled(inode, index, write_end, shmem_huge_force, vma, vm_flags); - /* Tmpfs huge pages allocation */ - if (!vma || !vma_is_anon_shmem(vma)) + /* + * Tmpfs huge pages allocation or forced collapse ignores + * sysfs configs. + */ + if (!vma || !vma_is_anon_shmem(vma) || shmem_huge_force) return global_orders; /* diff --git a/mm/slub.c b/mm/slub.c index 54ec12503357..544cff39762c 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -5969,7 +5969,8 @@ __pcs_replace_full_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, if (!allow_spin) return NULL; - empty = alloc_empty_sheaf(s, GFP_NOWAIT, SLAB_ALLOC_DEFAULT); + /* Don't wake up kswapd, it will cause deadlock under pi_lock */ + empty = alloc_empty_sheaf(s, __GFP_NOWARN, SLAB_ALLOC_DEFAULT); if (empty) goto got_empty; @@ -6128,7 +6129,6 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags) struct slab_sheaf *empty; struct node_barn *barn; unsigned int alloc_flags = to_alloc_flags(free_flags); - gfp_t gfp = allow_spin ? GFP_NOWAIT : __GFP_NOWARN; /* Bootstrap or debug cache, fall back */ if (unlikely(!cache_has_sheaves(s))) { @@ -6157,7 +6157,8 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags) local_unlock(&s->cpu_sheaves->lock); - empty = alloc_empty_sheaf(s, gfp, alloc_flags); + /* Don't wake up kswapd, it will cause deadlock under pi_lock */ + empty = alloc_empty_sheaf(s, __GFP_NOWARN, alloc_flags); if (!empty) goto fail; diff --git a/mm/vma.c b/mm/vma.c index f29abb30956b..6cde67883fb0 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -2094,6 +2094,13 @@ static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct * * acceptable for merging, so we can do all of this optimistically. But * we do that READ_ONCE() to make sure that we never re-load the pointer. * + * The READ_ONCE() establishes an address dependency between anon_vma and + * any access to its fields, which pairs with the assignment to + * vma->anon_vma performed with release semantics in __anon_vma_prepare(). + * + * This is especially important as anon_vma's are SLAB_TYPESAFE_BY_RCU so + * accessing an uninitialised anon_vma's fields may result in a UAF. + * * IOW: that the "list_is_singular()" test on the anon_vma_chain only * matters for the 'stable anon_vma' case (ie the thing we want to avoid * is to return an anon_vma that is "complex" due to having gone through @@ -2108,6 +2115,7 @@ static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_area_struct *b) { if (anon_vma_compatible(a, b)) { + /* Paired with a memory barrier in __anon_vma_prepare(). */ struct anon_vma *anon_vma = READ_ONCE(old->anon_vma); if (anon_vma && list_is_singular(&old->anon_vma_chain)) @@ -2841,7 +2849,7 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, { struct mm_struct *mm = current->mm; struct vm_area_struct *vma = NULL; - bool have_mmap_prepare = file && file->f_op->mmap_prepare; + const bool have_mmap_prepare = file && file->f_op->mmap_prepare; VMA_ITERATOR(vmi, mm, addr); const pgoff_t anon_pgoff = addr >> PAGE_SHIFT; MMAP_STATE(map, mm, &vmi, addr, len, pgoff, anon_pgoff, vma_flags, file); @@ -2884,7 +2892,7 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, allocated_new = true; } - if (have_mmap_prepare) + if (have_mmap_prepare && allocated_new) set_vma_user_defined_fields(vma, &map); __mmap_complete(&map, vma); diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c index 2859cbac3f26..c949c6a82945 100644 --- a/net/8021q/vlan_dev.c +++ b/net/8021q/vlan_dev.c @@ -55,6 +55,11 @@ static int vlan_dev_hard_header(struct sk_buff *skb, struct net_device *dev, int rc; if (!(vlan->flags & VLAN_FLAG_REORDER_HDR)) { + unsigned int hlen = READ_ONCE(dev->hard_header_len) + + READ_ONCE(dev->needed_headroom); + + if (skb_cow_head(skb, hlen) < 0) + return -ENOMEM; vhdr = skb_push(skb, VLAN_HLEN); vlan_tci = vlan->vlan_id; diff --git a/net/bluetooth/bnep/core.c b/net/bluetooth/bnep/core.c index f7d88c33e23e..ad24d2486665 100644 --- a/net/bluetooth/bnep/core.c +++ b/net/bluetooth/bnep/core.c @@ -270,9 +270,14 @@ static int bnep_rx_extension(struct bnep_session *s, struct sk_buff *skb) BT_DBG("type 0x%x len %u", h->type, h->len); + if (skb->len < h->len) { + err = -EILSEQ; + break; + } + switch (h->type & BNEP_TYPE_MASK) { case BNEP_EXT_CONTROL: - bnep_rx_control(s, skb->data, skb->len); + bnep_rx_control(s, skb->data, h->len); break; default: @@ -373,6 +378,11 @@ static int bnep_rx_frame(struct bnep_session *s, struct sk_buff *skb) goto badframe; } + if ((type & BNEP_TYPE_MASK) == BNEP_CONTROL) { + kfree_skb(skb); + return 0; + } + /* Strip 802.1p header */ if (ntohs(s->eh.h_proto) == ETH_P_8021Q) { if (!skb_pull(skb, 4)) @@ -451,6 +461,11 @@ static int bnep_tx_frame(struct bnep_session *s, struct sk_buff *skb) goto send; } + if (skb->len < ETH_HLEN) { + kfree_skb(skb); + return 0; + } + iv[il++] = (struct kvec) { &type, 1 }; len++; diff --git a/net/bluetooth/bnep/netdev.c b/net/bluetooth/bnep/netdev.c index ee1e39a3daff..b451ef457741 100644 --- a/net/bluetooth/bnep/netdev.c +++ b/net/bluetooth/bnep/netdev.c @@ -166,6 +166,12 @@ static netdev_tx_t bnep_net_xmit(struct sk_buff *skb, BT_DBG("skb %p, dev %p", skb, dev); + if (!pskb_may_pull(skb, ETH_HLEN)) { + dev->stats.tx_dropped++; + kfree_skb(skb); + return NETDEV_TX_OK; + } + #ifdef CONFIG_BT_BNEP_MC_FILTER if (bnep_net_mc_filter(skb, s)) { kfree_skb(skb); @@ -218,7 +224,7 @@ void bnep_net_setup(struct net_device *dev) dev->addr_len = ETH_ALEN; ether_setup(dev); - dev->min_mtu = 0; + dev->min_mtu = ETH_MIN_MTU; dev->max_mtu = ETH_MAX_MTU; dev->priv_flags &= ~IFF_TX_SKB_SHARING; dev->netdev_ops = &bnep_netdev_ops; diff --git a/net/bluetooth/hci_conn.c b/net/bluetooth/hci_conn.c index fa72cf8aaa7a..cf44452e0766 100644 --- a/net/bluetooth/hci_conn.c +++ b/net/bluetooth/hci_conn.c @@ -302,11 +302,13 @@ static int hci_enhanced_setup_sync(struct hci_dev *hdev, void *data) cp.tx_bandwidth = cpu_to_le32(0x00001f40); cp.rx_bandwidth = cpu_to_le32(0x00001f40); + hci_dev_lock(hdev); + switch (conn->codec.id) { case BT_CODEC_MSBC: if (!find_next_esco_param(conn, esco_param_msbc, ARRAY_SIZE(esco_param_msbc))) - return -EINVAL; + goto unlock; param = &esco_param_msbc[conn->attempt - 1]; cp.tx_coding_format.id = 0x05; @@ -332,7 +334,7 @@ static int hci_enhanced_setup_sync(struct hci_dev *hdev, void *data) case BT_CODEC_TRANSPARENT: if (!find_next_esco_param(conn, esco_param_msbc, ARRAY_SIZE(esco_param_msbc))) - return -EINVAL; + goto unlock; param = &esco_param_msbc[conn->attempt - 1]; cp.tx_coding_format.id = 0x03; @@ -359,11 +361,11 @@ static int hci_enhanced_setup_sync(struct hci_dev *hdev, void *data) if (conn->parent && lmp_esco_capable(conn->parent)) { if (!find_next_esco_param(conn, esco_param_cvsd, ARRAY_SIZE(esco_param_cvsd))) - return -EINVAL; + goto unlock; param = &esco_param_cvsd[conn->attempt - 1]; } else { if (conn->attempt > ARRAY_SIZE(sco_param_cvsd)) - return -EINVAL; + goto unlock; param = &sco_param_cvsd[conn->attempt - 1]; } cp.tx_coding_format.id = 2; @@ -386,9 +388,11 @@ static int hci_enhanced_setup_sync(struct hci_dev *hdev, void *data) cp.out_transport_unit_size = 16; break; default: - return -EINVAL; + goto unlock; } + hci_dev_unlock(hdev); + cp.retrans_effort = param->retrans_effort; cp.pkt_type = __cpu_to_le16(param->pkt_type); cp.max_latency = __cpu_to_le16(param->max_latency); @@ -397,6 +401,10 @@ static int hci_enhanced_setup_sync(struct hci_dev *hdev, void *data) return -EIO; return 0; + +unlock: + hci_dev_unlock(hdev); + return -EINVAL; } static bool hci_setup_sync_conn(struct hci_conn *conn, __u16 handle) @@ -2079,6 +2087,8 @@ struct hci_conn *hci_bind_cis(struct hci_dev *hdev, bdaddr_t *dst, cis->conn_timeout = timeout; } + hci_conn_hold(cis); + if (cis->state == BT_CONNECTED) return cis; @@ -2120,7 +2130,6 @@ struct hci_conn *hci_bind_cis(struct hci_dev *hdev, bdaddr_t *dst, return ERR_PTR(-EINVAL); } - hci_conn_hold(cis); cis->state = BT_BOUND; return cis; @@ -2374,10 +2383,13 @@ struct hci_conn *hci_bind_bis(struct hci_dev *hdev, bdaddr_t *dst, __u8 sid, parent = hci_conn_hash_lookup_big(hdev, conn->iso_qos.bcast.big); if (parent && parent != conn) { + hci_conn_hold(parent); link = hci_conn_link(parent, conn); hci_conn_drop(conn); - if (!link) + if (!link) { + hci_conn_drop(parent); return ERR_PTR(-ENOLINK); + } } return conn; @@ -2497,6 +2509,12 @@ struct hci_conn *hci_connect_cis(struct hci_dev *hdev, bdaddr_t *dst, return cis; } + /* The existing link already owns the hold on its parent. */ + if (cis->link) { + hci_conn_drop(le); + return cis; + } + link = hci_conn_link(le, cis); hci_conn_drop(cis); if (!link) { diff --git a/net/bluetooth/hci_core.c b/net/bluetooth/hci_core.c index d183efaf9063..2076689eb302 100644 --- a/net/bluetooth/hci_core.c +++ b/net/bluetooth/hci_core.c @@ -1489,8 +1489,10 @@ int hci_remove_remote_oob_data(struct hci_dev *hdev, bdaddr_t *bdaddr, BT_DBG("%s removing %pMR (%u)", hdev->name, bdaddr, bdaddr_type); + mutex_lock(&hdev->remote_oob_lock); list_del(&data->list); kfree(data); + mutex_unlock(&hdev->remote_oob_lock); return 0; } @@ -1499,10 +1501,12 @@ void hci_remote_oob_data_clear(struct hci_dev *hdev) { struct oob_data *data, *n; + mutex_lock(&hdev->remote_oob_lock); list_for_each_entry_safe(data, n, &hdev->remote_oob_data, list) { list_del(&data->list); kfree(data); } + mutex_unlock(&hdev->remote_oob_lock); } int hci_add_remote_oob_data(struct hci_dev *hdev, bdaddr_t *bdaddr, @@ -1511,11 +1515,14 @@ int hci_add_remote_oob_data(struct hci_dev *hdev, bdaddr_t *bdaddr, { struct oob_data *data; + mutex_lock(&hdev->remote_oob_lock); data = hci_find_remote_oob_data(hdev, bdaddr, bdaddr_type); if (!data) { data = kmalloc_obj(*data); - if (!data) + if (!data) { + mutex_unlock(&hdev->remote_oob_lock); return -ENOMEM; + } bacpy(&data->bdaddr, bdaddr); data->bdaddr_type = bdaddr_type; @@ -1548,6 +1555,8 @@ int hci_add_remote_oob_data(struct hci_dev *hdev, bdaddr_t *bdaddr, BT_DBG("%s for %pMR", hdev->name, bdaddr); + mutex_unlock(&hdev->remote_oob_lock); + return 0; } @@ -2485,6 +2494,7 @@ struct hci_dev *hci_alloc_dev_priv(int sizeof_priv) mutex_init(&hdev->lock); mutex_init(&hdev->req_lock); mutex_init(&hdev->mgmt_pending_lock); + mutex_init(&hdev->remote_oob_lock); ida_init(&hdev->unset_handle_ida); @@ -2557,8 +2567,10 @@ int hci_register_dev(struct hci_dev *hdev) return id; error = dev_set_name(&hdev->dev, "hci%u", id); - if (error) + if (error) { + ida_free(&hci_index_ida, id); return error; + } hdev->name = dev_name(&hdev->dev); hdev->id = id; @@ -3325,6 +3337,8 @@ static void hci_queue_iso(struct hci_conn *conn, struct sk_buff_head *queue, skb_shinfo(skb)->frag_list = NULL; + spin_lock_bh(&queue->lock); + __skb_queue_tail(queue, skb); do { @@ -3339,6 +3353,8 @@ static void hci_queue_iso(struct hci_conn *conn, struct sk_buff_head *queue, __skb_queue_tail(queue, skb); } while (list); + + spin_unlock_bh(&queue->lock); } bt_dev_dbg(hdev, "hcon %p queued %d", conn, skb_queue_len(queue)); @@ -3402,9 +3418,6 @@ static struct hci_conn *hci_low_sent(struct hci_dev *hdev, __u8 type, struct hci_conn *conn = NULL, *c; unsigned int num = 0, min = ~0; - /* We don't have to lock device here. Connections are always - * added and removed with TX task disabled. */ - rcu_read_lock(); list_for_each_entry_rcu(c, &h->list, list) { @@ -3609,13 +3622,15 @@ static void __check_timeout(struct hci_dev *hdev, unsigned int cnt, u8 type) } /* Schedule SCO */ -static void hci_sched_sco(struct hci_dev *hdev, __u8 type) +static void __hci_sched_sco(struct hci_dev *hdev, __u8 type) { struct hci_conn *conn; struct sk_buff *skb; int quote, *cnt; unsigned int pkts = hdev->sco_pkts; + lockdep_assert_held(&hdev->lock); + bt_dev_dbg(hdev, "type %u", type); if (!hci_conn_num(hdev, type) || !pkts) @@ -3650,6 +3665,13 @@ static void hci_sched_sco(struct hci_dev *hdev, __u8 type) queue_work(hdev->workqueue, &hdev->tx_work); } +static void hci_sched_sco(struct hci_dev *hdev, __u8 type) +{ + hci_dev_lock(hdev); + __hci_sched_sco(hdev, type); + hci_dev_unlock(hdev); +} + static void hci_sched_acl_pkt(struct hci_dev *hdev) { unsigned int cnt = hdev->acl_cnt; @@ -3659,6 +3681,8 @@ static void hci_sched_acl_pkt(struct hci_dev *hdev) __check_timeout(hdev, cnt, ACL_LINK); + hci_dev_lock(hdev); + while (hdev->acl_cnt && (chan = hci_chan_sent(hdev, ACL_LINK, "e))) { u32 priority = (skb_peek(&chan->data_q))->priority; @@ -3683,13 +3707,15 @@ static void hci_sched_acl_pkt(struct hci_dev *hdev) chan->conn->sent++; /* Send pending SCO packets right away */ - hci_sched_sco(hdev, SCO_LINK); - hci_sched_sco(hdev, ESCO_LINK); + __hci_sched_sco(hdev, SCO_LINK); + __hci_sched_sco(hdev, ESCO_LINK); } } if (cnt != hdev->acl_cnt) hci_prio_recalculate(hdev, ACL_LINK); + + hci_dev_unlock(hdev); } static void hci_sched_acl(struct hci_dev *hdev) @@ -3718,6 +3744,8 @@ static void hci_sched_le(struct hci_dev *hdev) __check_timeout(hdev, *cnt, LE_LINK); + hci_dev_lock(hdev); + tmp = *cnt; while (*cnt && (chan = hci_chan_sent(hdev, LE_LINK, "e))) { u32 priority = (skb_peek(&chan->data_q))->priority; @@ -3739,13 +3767,15 @@ static void hci_sched_le(struct hci_dev *hdev) chan->conn->sent++; /* Send pending SCO packets right away */ - hci_sched_sco(hdev, SCO_LINK); - hci_sched_sco(hdev, ESCO_LINK); + __hci_sched_sco(hdev, SCO_LINK); + __hci_sched_sco(hdev, ESCO_LINK); } } if (*cnt != tmp) hci_prio_recalculate(hdev, LE_LINK); + + hci_dev_unlock(hdev); } /* Schedule iso */ @@ -3764,6 +3794,8 @@ static void hci_sched_iso(struct hci_dev *hdev, __u8 type) __check_timeout(hdev, *cnt, type); + hci_dev_lock(hdev); + while (*cnt && (conn = hci_low_sent(hdev, type, "e))) { while (quote-- && (skb = skb_dequeue(&conn->data_q))) { BT_DBG("skb %p len %d", skb, skb->len); @@ -3777,6 +3809,8 @@ static void hci_sched_iso(struct hci_dev *hdev, __u8 type) (*cnt)--; } } + + hci_dev_unlock(hdev); } static void hci_tx_work(struct work_struct *work) diff --git a/net/bluetooth/hci_debugfs.c b/net/bluetooth/hci_debugfs.c index aadffaaff20e..2559fb6324d6 100644 --- a/net/bluetooth/hci_debugfs.c +++ b/net/bluetooth/hci_debugfs.c @@ -364,7 +364,7 @@ static int inquiry_cache_show(struct seq_file *f, void *p) list_for_each_entry(e, &cache->all, all) { struct inquiry_data *data = &e->data; - seq_printf(f, "%pMR %d %d %d 0x%.2x%.2x%.2x 0x%.4x %d %d %u\n", + seq_printf(f, "%pMR %d %d %d 0x%.2x%.2x%.2x 0x%.4x %d %d %lu\n", &data->bdaddr, data->pscan_rep_mode, data->pscan_period_mode, data->pscan_mode, data->dev_class[2], diff --git a/net/bluetooth/hci_sock.c b/net/bluetooth/hci_sock.c index 070ca388f9ac..6d56c77741e1 100644 --- a/net/bluetooth/hci_sock.c +++ b/net/bluetooth/hci_sock.c @@ -164,6 +164,7 @@ static bool is_filtered_packet(struct sock *sk, struct sk_buff *skb) { struct hci_filter *flt; int flt_type, flt_event; + u8 event; /* Apply filter */ flt = &hci_pi(sk)->filter; @@ -177,7 +178,11 @@ static bool is_filtered_packet(struct sock *sk, struct sk_buff *skb) if (hci_skb_pkt_type(skb) != HCI_EVENT_PKT) return false; - flt_event = (*(__u8 *)skb->data & HCI_FLT_EVENT_BITS); + if (skb->len < 1) + return true; + + event = *(__u8 *)skb->data; + flt_event = event & HCI_FLT_EVENT_BITS; if (!hci_test_bit(flt_event, &flt->event_mask)) return true; @@ -186,11 +191,17 @@ static bool is_filtered_packet(struct sock *sk, struct sk_buff *skb) if (!flt->opcode) return false; - if (flt_event == HCI_EV_CMD_COMPLETE && + if (event == HCI_EV_CMD_COMPLETE && skb->len < 5) + return true; + + if (event == HCI_EV_CMD_COMPLETE && flt->opcode != get_unaligned((__le16 *)(skb->data + 3))) return true; - if (flt_event == HCI_EV_CMD_STATUS && + if (event == HCI_EV_CMD_STATUS && skb->len < 6) + return true; + + if (event == HCI_EV_CMD_STATUS && flt->opcode != get_unaligned((__le16 *)(skb->data + 4))) return true; @@ -1881,7 +1892,8 @@ static int hci_sock_sendmsg(struct socket *sock, struct msghdr *msg, u16 ocf = hci_opcode_ocf(opcode); if (((ogf > HCI_SFLT_MAX_OGF) || - !hci_test_bit(ocf & HCI_FLT_OCF_BITS, + (ocf > HCI_FLT_OCF_BITS) || + !hci_test_bit(ocf, &hci_sec_filter.ocf_mask[ogf])) && !capable(CAP_NET_RAW)) { err = -EPERM; diff --git a/net/bluetooth/hci_sync.c b/net/bluetooth/hci_sync.c index 74e2b04c84b2..c01c8b58d9e8 100644 --- a/net/bluetooth/hci_sync.c +++ b/net/bluetooth/hci_sync.c @@ -312,6 +312,10 @@ static void hci_cmd_sync_work(struct work_struct *work) while (1) { struct hci_cmd_sync_work_entry *entry; + /* Leave the backlog to hci_cmd_sync_clear() */ + if (hci_dev_test_flag(hdev, HCI_UNREGISTER)) + break; + mutex_lock(&hdev->cmd_sync_work_lock); entry = list_first_entry_or_null(&hdev->cmd_sync_work_list, struct hci_cmd_sync_work_entry, @@ -658,6 +662,8 @@ void hci_cmd_sync_clear(struct hci_dev *hdev) { struct hci_cmd_sync_work_entry *entry, *tmp; + /* cmd_work is disabled, the pending request can only time out */ + hci_cmd_sync_cancel_sync(hdev, ENODEV); cancel_work_sync(&hdev->cmd_sync_work); cancel_work_sync(&hdev->reenable_adv_work); @@ -5764,6 +5770,7 @@ int hci_stop_discovery_sync(struct hci_dev *hdev) { struct discovery_state *d = &hdev->discovery; struct inquiry_entry *e; + bdaddr_t addr; int err; bt_dev_dbg(hdev, "state %u", hdev->discovery.state); @@ -5799,15 +5806,21 @@ int hci_stop_discovery_sync(struct hci_dev *hdev) return 0; if (d->state == DISCOVERY_RESOLVING || d->state == DISCOVERY_STOPPING) { + hci_dev_lock(hdev); e = hci_inquiry_cache_lookup_resolve(hdev, BDADDR_ANY, NAME_PENDING); - if (!e) + if (!e) { + hci_dev_unlock(hdev); return 0; + } + + bacpy(&addr, &e->data.bdaddr); + hci_dev_unlock(hdev); /* Ignore cancel errors since it should interfere with stopping * of the discovery. */ - hci_remote_name_cancel_sync(hdev, &e->data.bdaddr); + hci_remote_name_cancel_sync(hdev, &addr); } return 0; @@ -7236,6 +7249,7 @@ static int hci_acl_create_conn_sync(struct hci_dev *hdev, void *data) bacpy(&cp.bdaddr, &conn->dst); cp.pscan_rep_mode = 0x02; + hci_dev_lock(hdev); ie = hci_inquiry_cache_lookup(hdev, &conn->dst); if (ie) { if (inquiry_entry_age(ie) <= INQUIRY_ENTRY_AGE_MAX) { @@ -7247,6 +7261,7 @@ static int hci_acl_create_conn_sync(struct hci_dev *hdev, void *data) memcpy(conn->dev_class, ie->data.dev_class, 3); } + hci_dev_unlock(hdev); cp.pkt_type = cpu_to_le16(conn->pkt_type); if (lmp_rswitch_capable(hdev) && !(hdev->link_mode & HCI_LM_MASTER)) diff --git a/net/bluetooth/iso.c b/net/bluetooth/iso.c index eb99653f33f9..7657c2a0abbf 100644 --- a/net/bluetooth/iso.c +++ b/net/bluetooth/iso.c @@ -496,6 +496,7 @@ static int iso_connect_cis(struct sock *sk) struct hci_dev *hdev; bdaddr_t src, dst; u8 src_type; + bool already_attached; int err; lock_sock(sk); @@ -568,8 +569,14 @@ static int iso_connect_cis(struct sock *sk) goto unlock; } + iso_conn_lock(conn); + already_attached = iso_pi(sk)->conn == conn && conn->sk == sk; + iso_conn_unlock(conn); + err = iso_chan_add(conn, sk, NULL); iso_conn_put(conn); + if (already_attached || err == -EBUSY) + hci_conn_drop(hcon); if (err) goto unlock; diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c index 644e31160d55..aaa2a1cd489a 100644 --- a/net/bluetooth/l2cap_core.c +++ b/net/bluetooth/l2cap_core.c @@ -6702,9 +6702,17 @@ static int l2cap_stream_rx(struct l2cap_chan *chan, struct l2cap_ctrl *control, static int l2cap_data_rcv(struct l2cap_chan *chan, struct sk_buff *skb) { struct l2cap_ctrl *control = &bt_cb(skb)->l2cap; - u16 len; + u16 len, min_len; u8 event; + min_len = test_bit(FLAG_EXT_CTRL, &chan->flags) ? + L2CAP_EXT_CTRL_SIZE : L2CAP_ENH_CTRL_SIZE; + if (chan->fcs == L2CAP_FCS_CRC16) + min_len += L2CAP_FCS_SIZE; + + if (skb->len < min_len) + goto drop; + __unpack_control(chan, skb); len = skb->len; diff --git a/net/bluetooth/mgmt.c b/net/bluetooth/mgmt.c index ac4864e56ec7..41956cdde982 100644 --- a/net/bluetooth/mgmt.c +++ b/net/bluetooth/mgmt.c @@ -496,13 +496,9 @@ static int read_unconf_index_list(struct sock *sk, struct hci_dev *hdev, read_lock(&hci_dev_list_lock); - count = 0; - list_for_each_entry(d, &hci_dev_list, list) { - if (hci_dev_test_flag(d, HCI_UNCONFIGURED)) - count++; - } + count = list_count_nodes(&hci_dev_list); - rp_len = sizeof(*rp) + (2 * count); + rp_len = sizeof(*rp) + (sizeof(__le16) * count); rp = kmalloc(rp_len, GFP_ATOMIC); if (!rp) { read_unlock(&hci_dev_list_lock); @@ -2316,6 +2312,8 @@ static void mesh_send_start_complete(struct hci_dev *hdev, void *data, int err) hci_dev_clear_flag(hdev, HCI_MESH_SENDING); /* Send Complete Error Code for handle */ mesh_send_complete(hdev, mesh_tx, false); + if (err != -ECANCELED) + mesh_next(hdev, NULL, 0); return; } @@ -2425,19 +2423,28 @@ static int send_cancel(struct hci_dev *hdev, void *data) do { mesh_tx = mgmt_mesh_next(hdev, cmd->sk); - if (mesh_tx) - mesh_send_complete(hdev, mesh_tx, false); + if (mesh_tx) { + if (!hci_cmd_sync_dequeue(hdev, mesh_send_sync, + mesh_tx, NULL)) + mesh_send_complete(hdev, mesh_tx, false); + } } while (mesh_tx); } else { mesh_tx = mgmt_mesh_find(hdev, cancel->handle); - if (mesh_tx && mesh_tx->sk == cmd->sk) - mesh_send_complete(hdev, mesh_tx, false); + if (mesh_tx && mesh_tx->sk == cmd->sk) { + if (!hci_cmd_sync_dequeue(hdev, mesh_send_sync, + mesh_tx, NULL)) + mesh_send_complete(hdev, mesh_tx, false); + } } mgmt_cmd_complete(cmd->sk, hdev->id, MGMT_OP_MESH_SEND_CANCEL, 0, NULL, 0); + if (!hci_dev_test_flag(hdev, HCI_MESH_SENDING)) + mesh_next(hdev, NULL, 0); + return 0; } diff --git a/net/bluetooth/rfcomm/core.c b/net/bluetooth/rfcomm/core.c index f7463f092283..d91e2a6ee26c 100644 --- a/net/bluetooth/rfcomm/core.c +++ b/net/bluetooth/rfcomm/core.c @@ -1817,7 +1817,8 @@ static struct rfcomm_session *rfcomm_recv_frame(struct rfcomm_session *s, return s; } - if (skb->len < sizeof(*hdr) + 1) { + if (skb->len < sizeof(*hdr) + 1 || + (!__test_ea(hdr->len) && skb->len < sizeof(*hdr) + 2)) { kfree_skb(skb); return s; } diff --git a/net/bluetooth/rfcomm/sock.c b/net/bluetooth/rfcomm/sock.c index e2486bc11cbc..fb924d0e34ec 100644 --- a/net/bluetooth/rfcomm/sock.c +++ b/net/bluetooth/rfcomm/sock.c @@ -786,8 +786,10 @@ static int rfcomm_sock_getsockopt_old(struct socket *sock, int optname, break; case RFCOMM_CONNINFO: - if (sk->sk_state != BT_CONNECTED && - !rfcomm_pi(sk)->dlc->defer_setup) { + if ((sk->sk_state != BT_CONNECTED && + !(sk->sk_state == BT_CONNECT2 && + rfcomm_pi(sk)->dlc->defer_setup)) || + !rfcomm_pi(sk)->dlc->session) { err = -ENOTCONN; break; } diff --git a/net/bluetooth/rfcomm/tty.c b/net/bluetooth/rfcomm/tty.c index b2c1060394e6..dc3cdf614def 100644 --- a/net/bluetooth/rfcomm/tty.c +++ b/net/bluetooth/rfcomm/tty.c @@ -462,7 +462,7 @@ static int __rfcomm_release_dev(void __user *arg) /* Shut down TTY synchronously before freeing rfcomm_dev */ tty_port_tty_vhangup(&dev->port); - if (!test_bit(RFCOMM_TTY_OWNED, &dev->status)) + if (!test_and_set_bit(RFCOMM_TTY_OWNED, &dev->status)) tty_port_put(&dev->port); tty_port_put(&dev->port); @@ -724,10 +724,9 @@ static int rfcomm_tty_install(struct tty_driver *driver, struct tty_struct *tty) * when the last process closes the tty. The behaviour is expected by * userspace. */ - if (test_bit(RFCOMM_RELEASE_ONHUP, &dev->flags)) { - set_bit(RFCOMM_TTY_OWNED, &dev->status); + if (test_bit(RFCOMM_RELEASE_ONHUP, &dev->flags) && + !test_and_set_bit(RFCOMM_TTY_OWNED, &dev->status)) tty_port_put(&dev->port); - } return 0; } diff --git a/net/bluetooth/smp.c b/net/bluetooth/smp.c index 6091c47cb002..2df303f6a38f 100644 --- a/net/bluetooth/smp.c +++ b/net/bluetooth/smp.c @@ -662,6 +662,7 @@ static void build_pairing_cmd(struct l2cap_conn *conn, else bdaddr_type = BDADDR_LE_RANDOM; + mutex_lock(&hdev->remote_oob_lock); oob_data = hci_find_remote_oob_data(hdev, &hcon->dst, bdaddr_type); if (oob_data && oob_data->present) { @@ -672,6 +673,7 @@ static void build_pairing_cmd(struct l2cap_conn *conn, SMP_DBG("OOB Remote Confirmation: %16phN", smp->pcnf); SMP_DBG("OOB Remote Random: %16phN", smp->rr); } + mutex_unlock(&hdev->remote_oob_lock); } else { authreq &= ~SMP_AUTH_SC; @@ -2269,6 +2271,23 @@ static u8 smp_cmd_security_req(struct l2cap_conn *conn, struct sk_buff *skb) bt_dev_dbg(hdev, "conn %p", conn); + /* SMP over BR/EDR only covers cross-transport key derivation; the + * Security Request procedure has no BR/EDR counterpart. Reject it + * here, otherwise smp_ltk_encrypt() finds the peer's LE LTK + * (ADDR_LE_DEV_PUBLIC and BDADDR_BREDR are both 0) and issues + * HCI_OP_LE_START_ENC on the ACL handle, which the controller + * rejects and hci_cs_le_start_enc() turns into a disconnect. Reply + * without smp_failure(): this is not an authentication failure, and + * MGMT_EV_AUTH_FAILED would make bluetoothd drop the device. + */ + if (hcon->type != LE_LINK) { + u8 reason = SMP_CMD_NOTSUPP; + + smp_send_cmd(conn, SMP_CMD_PAIRING_FAIL, sizeof(reason), + &reason); + return 0; + } + if (skb->len < sizeof(*rp)) return SMP_INVALID_PARAMS; diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index 5d51f6cb7d15..513354e928cb 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -205,8 +205,8 @@ static void xdp_test_run_teardown(struct xdp_test_data *xdp) { xdp_unreg_mem_model(&xdp->mem); page_pool_destroy(xdp->pp); - kfree(xdp->frames); - kfree(xdp->skbs); + kvfree(xdp->frames); + kvfree(xdp->skbs); } static bool frame_was_changed(const struct xdp_page_head *head) diff --git a/net/bridge/br_mdb.c b/net/bridge/br_mdb.c index e0c7020b12f5..a01bd280c722 100644 --- a/net/bridge/br_mdb.c +++ b/net/bridge/br_mdb.c @@ -1523,6 +1523,8 @@ static void br_mdb_flush_pgs(struct net_bridge *br, } br_multicast_del_pg(mp, p, pp); + /* br_multicast_del_pg() can remove other groups from this list. */ + pp = &mp->ports; } } diff --git a/net/bridge/br_stp_bpdu.c b/net/bridge/br_stp_bpdu.c index 74ec42ba1e7d..21d092f5acbb 100644 --- a/net/bridge/br_stp_bpdu.c +++ b/net/bridge/br_stp_bpdu.c @@ -52,7 +52,10 @@ static void br_send_bpdu(struct net_bridge_port *p, LLC_SAP_BSPAN, LLC_PDU_CMD); llc_pdu_init_as_ui_cmd(skb); - llc_mac_hdr_init(skb, p->dev->dev_addr, p->br->group_addr); + if (llc_mac_hdr_init(skb, p->dev->dev_addr, p->br->group_addr)) { + kfree_skb(skb); + return; + } skb_reset_mac_header(skb); diff --git a/net/core/dev.c b/net/core/dev.c index c67900354fa6..18dc88990510 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -2901,7 +2901,7 @@ int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask, dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev; tc = netdev_txq_to_tc(dev, index); - if (tc < 0) + if (tc < 0 || tc >= num_tc) return -EINVAL; } @@ -3818,20 +3818,36 @@ static netdev_features_t dflt_features_check(struct sk_buff *skb, return vlan_features_check(skb, features); } -static bool skb_gso_has_extension_hdr(const struct sk_buff *skb) +static bool __skb_has_ipv6_ext_hdr(const struct sk_buff *skb, int nhoff) { - if (!skb->encapsulation) - return ((skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6 || - (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4 && - vlan_get_protocol(skb) == htons(ETH_P_IPV6))) && - skb_transport_header_was_set(skb) && - skb_network_header_len(skb) != sizeof(struct ipv6hdr)); - else - return (!skb_inner_network_header_was_set(skb) || - ((skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6 || - (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4 && - inner_ip_hdr(skb)->version == 6)) && - skb_inner_network_header_len(skb) != sizeof(struct ipv6hdr))); + const struct ipv6hdr *ip6h; + struct ipv6hdr _ip6h; + + ip6h = skb_header_pointer(skb, nhoff, sizeof(_ip6h), &_ip6h); + return ip6h && ip6h->version == 6 && ipv6_ext_hdr(ip6h->nexthdr); +} + +static bool skb_has_ipv6_extension_hdr(const struct sk_buff *skb) +{ + if (vlan_get_protocol(skb) == htons(ETH_P_IPV6)) { + if (__skb_has_ipv6_ext_hdr(skb, skb_network_offset(skb))) + return true; + + /* Catch tunnels without skb->encapsulation (e.g., virtio). */ + if (!skb->encapsulation && + skb_transport_header_was_set(skb) && + skb_network_header_len(skb) != sizeof(struct ipv6hdr)) + return true; + } + + /* Tunnels without an inner network header, such as SCTP-in-UDP or + * PSP, have no inner IP header and thus no inner extension header. + */ + if (skb->encapsulation && skb_inner_network_header_was_set(skb) && + __skb_has_ipv6_ext_hdr(skb, skb_inner_network_offset(skb))) + return true; + + return false; } static netdev_features_t gso_features_check(const struct sk_buff *skb, @@ -3886,7 +3902,7 @@ static netdev_features_t gso_features_check(const struct sk_buff *skb, * so neither does TSO that depends on it. */ if (features & NETIF_F_IPV6_CSUM && - skb_gso_has_extension_hdr(skb)) + skb_has_ipv6_extension_hdr(skb)) features &= ~(NETIF_F_IPV6_CSUM | NETIF_F_TSO6 | NETIF_F_GSO_UDP_L4); return features; @@ -3988,8 +4004,7 @@ int skb_csum_hwoffload_help(struct sk_buff *skb, return 0; if (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) { - if (vlan_get_protocol(skb) == htons(ETH_P_IPV6) && - skb_network_header_len(skb) != sizeof(struct ipv6hdr)) + if (skb_has_ipv6_extension_hdr(skb)) goto sw_checksum; switch (skb->csum_offset) { @@ -4404,9 +4419,14 @@ EXPORT_SYMBOL(dev_loopback_xmit); static struct netdev_queue * netdev_tx_queue_mapping(struct net_device *dev, struct sk_buff *skb) { - int qm = skb_get_queue_mapping(skb); + int queue = skb_get_queue_mapping(skb); + int capped; - return netdev_get_tx_queue(dev, netdev_cap_txqueue(dev, qm)); + capped = netdev_cap_txqueue(dev, queue); + if (unlikely(capped != queue)) + skb_set_queue_mapping(skb, capped); + + return netdev_get_tx_queue(dev, capped); } #ifndef CONFIG_PREEMPT_RT @@ -4415,9 +4435,13 @@ static bool netdev_xmit_txqueue_skipped(void) return __this_cpu_read(softnet_data.xmit.skip_txqueue); } -void netdev_xmit_skip_txqueue(bool skip) +bool netdev_xmit_skip_txqueue(bool skip) { + bool prev = netdev_xmit_txqueue_skipped(); + __this_cpu_write(softnet_data.xmit.skip_txqueue, skip); + + return prev; } EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue); @@ -4427,9 +4451,13 @@ static bool netdev_xmit_txqueue_skipped(void) return current->net_xmit.skip_txqueue; } -void netdev_xmit_skip_txqueue(bool skip) +bool netdev_xmit_skip_txqueue(bool skip) { + bool prev = netdev_xmit_txqueue_skipped(); + current->net_xmit.skip_txqueue = skip; + + return prev; } EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue); #endif @@ -4848,21 +4876,25 @@ int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) tcx_set_ingress(skb, false); #ifdef CONFIG_NET_EGRESS if (static_branch_unlikely(&egress_needed_key)) { + bool skip_txq; + if (nf_hook_egress_active()) { skb = nf_hook_egress(skb, &rc, dev); if (!skb) goto out; } - netdev_xmit_skip_txqueue(false); + skip_txq = netdev_xmit_skip_txqueue(false); nf_skip_egress(skb, true); skb = sch_handle_egress(skb, &rc, dev); - if (!skb) + if (!skb) { + netdev_xmit_skip_txqueue(skip_txq); goto out; + } nf_skip_egress(skb, false); - if (netdev_xmit_txqueue_skipped()) + if (netdev_xmit_skip_txqueue(skip_txq)) txq = netdev_tx_queue_mapping(dev, skb); } #endif @@ -5376,7 +5408,8 @@ void kick_defer_list_purge(unsigned int cpu) backlog_unlock_irq_restore(sd, flags); } else if (!cmpxchg(&sd->defer_ipi_scheduled, 0, 1)) { - smp_call_function_single_async(cpu, &sd->defer_csd); + if (smp_call_function_single_async(cpu, &sd->defer_csd)) + WRITE_ONCE(sd->defer_ipi_scheduled, 0); } } @@ -6900,25 +6933,35 @@ bool napi_complete_done(struct napi_struct *n, int work_done) } EXPORT_SYMBOL(napi_complete_done); -static void skb_defer_free_flush(void) +static void __skb_defer_free_flush(struct skb_defer_node *sdn, int budget) { struct llist_node *free_list; struct sk_buff *skb, *next; + + if (llist_empty(&sdn->defer_list)) + return; + atomic_long_set(&sdn->defer_count, 0); + free_list = llist_del_all(&sdn->defer_list); + + llist_for_each_entry_safe(skb, next, free_list, ll_node) { + prefetch(next); + napi_consume_skb(skb, budget); + } +} + +void skb_defer_node_flush(struct skb_defer_node *sdn) +{ + __skb_defer_free_flush(sdn, 0); +} + +static void skb_defer_free_flush(void) +{ struct skb_defer_node *sdn; int node; for_each_node(node) { sdn = this_cpu_ptr(net_hotdata.skb_defer_nodes) + node; - - if (llist_empty(&sdn->defer_list)) - continue; - atomic_long_set(&sdn->defer_count, 0); - free_list = llist_del_all(&sdn->defer_list); - - llist_for_each_entry_safe(skb, next, free_list, ll_node) { - prefetch(next); - napi_consume_skb(skb, 1); - } + __skb_defer_free_flush(sdn, 1); } } @@ -12897,6 +12940,7 @@ static int dev_cpu_dead(unsigned int oldcpu) struct sk_buff **list_skb; struct sk_buff *skb; unsigned int cpu; + int node; struct softnet_data *sd, *oldsd, *remsd = NULL; local_irq_disable(); @@ -12957,6 +13001,17 @@ static int dev_cpu_dead(unsigned int oldcpu) rps_input_queue_head_incr(oldsd); } + for_each_node(node) + skb_defer_node_flush(per_cpu_ptr(net_hotdata.skb_defer_nodes, + oldcpu) + node); + node = cpu_to_node(oldcpu); + if (node_possible(node) && + !cpumask_intersects(cpumask_of_node(node), cpu_online_mask)) { + for_each_possible_cpu(cpu) + skb_defer_node_flush(per_cpu_ptr(net_hotdata.skb_defer_nodes, + cpu) + node); + } + return 0; } diff --git a/net/core/dev.h b/net/core/dev.h index b757faead4d1..04fb0e9a571e 100644 --- a/net/core/dev.h +++ b/net/core/dev.h @@ -399,6 +399,8 @@ static inline void napi_assert_will_not_race(const struct napi_struct *napi) WARN_ON(READ_ONCE(napi->list_owner) != -1); } +struct skb_defer_node; +void skb_defer_node_flush(struct skb_defer_node *sdn); void kick_defer_list_purge(unsigned int cpu); int dev_set_hwtstamp_phylib(struct net_device *dev, diff --git a/net/core/dev_ioctl.c b/net/core/dev_ioctl.c index a320e264eaaf..f6029f60c1dc 100644 --- a/net/core/dev_ioctl.c +++ b/net/core/dev_ioctl.c @@ -267,7 +267,8 @@ int dev_get_hwtstamp_phylib(struct net_device *dev, hwprov->phydev) return phy_hwtstamp_get(hwprov->phydev, cfg); - if (hwprov->source == HWTSTAMP_SOURCE_NETDEV) + if (hwprov->source == HWTSTAMP_SOURCE_NETDEV && + dev->netdev_ops->ndo_hwtstamp_get) return dev->netdev_ops->ndo_hwtstamp_get(dev, cfg); return -EOPNOTSUPP; @@ -276,19 +277,18 @@ int dev_get_hwtstamp_phylib(struct net_device *dev, if (phy_is_default_hwtstamp(dev->phydev)) return phy_hwtstamp_get(dev->phydev, cfg); + if (!dev->netdev_ops->ndo_hwtstamp_get) + return -EOPNOTSUPP; + return dev->netdev_ops->ndo_hwtstamp_get(dev, cfg); } static int dev_get_hwtstamp(struct net_device *dev, struct ifreq *ifr) { - const struct net_device_ops *ops = dev->netdev_ops; struct kernel_hwtstamp_config kernel_cfg = {}; struct hwtstamp_config cfg; int err; - if (!ops->ndo_hwtstamp_get) - return -EOPNOTSUPP; - if (!netif_device_present(dev)) return -ENODEV; @@ -359,12 +359,18 @@ int dev_set_hwtstamp_phylib(struct net_device *dev, cfg->source = phy_ts ? HWTSTAMP_SOURCE_PHYLIB : HWTSTAMP_SOURCE_NETDEV; if (phy_ts && dev->see_all_hwtstamp_requests) { + if (!ops->ndo_hwtstamp_get) + return -EOPNOTSUPP; + err = ops->ndo_hwtstamp_get(dev, &old_cfg); if (err) return err; } if (!phy_ts || dev->see_all_hwtstamp_requests) { + if (!ops->ndo_hwtstamp_set) + return -EOPNOTSUPP; + err = ops->ndo_hwtstamp_set(dev, cfg, extack); if (err) { if (extack->_msg) @@ -390,7 +396,6 @@ int dev_set_hwtstamp_phylib(struct net_device *dev, static int dev_set_hwtstamp(struct net_device *dev, struct ifreq *ifr) { - const struct net_device_ops *ops = dev->netdev_ops; struct kernel_hwtstamp_config kernel_cfg = {}; struct netlink_ext_ack extack = {}; struct hwtstamp_config cfg; @@ -413,9 +418,6 @@ static int dev_set_hwtstamp(struct net_device *dev, struct ifreq *ifr) return err; } - if (!ops->ndo_hwtstamp_set) - return -EOPNOTSUPP; - if (!netif_device_present(dev)) return -ENODEV; @@ -441,15 +443,11 @@ static int dev_set_hwtstamp(struct net_device *dev, struct ifreq *ifr) int generic_hwtstamp_get_lower(struct net_device *dev, struct kernel_hwtstamp_config *kernel_cfg) { - const struct net_device_ops *ops = dev->netdev_ops; int err; if (!netif_device_present(dev)) return -ENODEV; - if (!ops->ndo_hwtstamp_get) - return -EOPNOTSUPP; - netdev_lock_ops(dev); err = dev_get_hwtstamp_phylib(dev, kernel_cfg); netdev_unlock_ops(dev); @@ -462,15 +460,11 @@ int generic_hwtstamp_set_lower(struct net_device *dev, struct kernel_hwtstamp_config *kernel_cfg, struct netlink_ext_ack *extack) { - const struct net_device_ops *ops = dev->netdev_ops; int err; if (!netif_device_present(dev)) return -ENODEV; - if (!ops->ndo_hwtstamp_set) - return -EOPNOTSUPP; - netdev_lock_ops(dev); err = dev_set_hwtstamp_phylib(dev, kernel_cfg, extack); netdev_unlock_ops(dev); diff --git a/net/core/filter.c b/net/core/filter.c index 61940e753552..70dc621672f2 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -3961,12 +3961,6 @@ static u32 __bpf_skb_min_len(const struct sk_buff *skb) if (offset > 0) min_len = offset; } - if (skb->ip_summed == CHECKSUM_PARTIAL) { - offset = skb_checksum_start_offset(skb) + - skb->csum_offset + sizeof(__sum16); - if (offset > 0) - min_len = offset; - } return min_len; } @@ -3983,6 +3977,11 @@ static int bpf_skb_grow_rcsum(struct sk_buff *skb, unsigned int new_len) static int bpf_skb_trim_rcsum(struct sk_buff *skb, unsigned int new_len) { + if (skb->ip_summed == CHECKSUM_PARTIAL && + new_len < skb_checksum_start_offset(skb) + skb->csum_offset + + sizeof(__sum16)) + skb->ip_summed = CHECKSUM_NONE; + return __skb_trim_rcsum(skb, new_len); } @@ -9045,6 +9044,8 @@ static const struct bpf_func_proto * lwt_seg6local_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { switch (func_id) { + case BPF_FUNC_skb_pull_data: + return NULL; #if IS_ENABLED(CONFIG_IPV6_SEG6_BPF) case BPF_FUNC_lwt_seg6_store_bytes: return &bpf_lwt_seg6_store_bytes_proto; @@ -10565,11 +10566,12 @@ u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, target_size)); *insn++ = BPF_JMP_IMM(BPF_JNE, si->dst_reg, NO_QUEUE_MAPPING, 1); - *insn++ = BPF_MOV64_IMM(si->dst_reg, -1); + *insn++ = BPF_MOV32_IMM(si->dst_reg, -1); #else - *insn++ = BPF_MOV64_IMM(si->dst_reg, -1); - *target_size = 2; + *insn++ = BPF_MOV32_IMM(si->dst_reg, -1); #endif + *target_size = sizeof_field(struct bpf_sock, rx_queue_mapping); + break; } @@ -11105,18 +11107,7 @@ static u32 sock_ops_convert_ctx_access(enum bpf_access_type type, break; case offsetof(struct bpf_sock_ops, rtt_min): - BUILD_BUG_ON(sizeof_field(struct tcp_sock, rtt_min) != - sizeof(struct minmax)); - BUILD_BUG_ON(sizeof(struct minmax) < - sizeof(struct minmax_sample)); - - *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF( - struct bpf_sock_ops_kern, sk), - si->dst_reg, si->src_reg, - offsetof(struct bpf_sock_ops_kern, sk)); - *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg, - offsetof(struct tcp_sock, rtt_min) + - sizeof_field(struct minmax_sample, t)); + SOCK_OPS_GET_FIELD(rtt_min, rtt_min.s[0].v, struct tcp_sock); break; case offsetof(struct bpf_sock_ops, bpf_sock_ops_cb_flags): @@ -12912,8 +12903,9 @@ __bpf_kfunc_start_defs(); * @sock: Pointer to socket to be destroyed * * Return: - * On error, may return EPROTONOSUPPORT, EINVAL. - * EPROTONOSUPPORT if protocol specific destroy handler is not supported. + * On error, may return EOPNOTSUPP, or whatever the protocol specific + * destroy handler returns. + * EOPNOTSUPP if protocol specific destroy handler is not supported. * 0 otherwise */ __bpf_kfunc int bpf_sock_destroy(struct sock_common *sock) @@ -12925,8 +12917,12 @@ __bpf_kfunc int bpf_sock_destroy(struct sock_common *sock) * Supporting protocols will need to acquire sock lock in the BPF context * prior to invoking this kfunc. */ - if (!sk->sk_prot->diag_destroy || (sk->sk_protocol != IPPROTO_TCP && - sk->sk_protocol != IPPROTO_UDP)) + if (!sk->sk_prot->diag_destroy) + return -EOPNOTSUPP; + + if (sk_fullsock(sk) && + sk->sk_protocol != IPPROTO_TCP && + sk->sk_protocol != IPPROTO_UDP) return -EOPNOTSUPP; return sk->sk_prot->diag_destroy(sk, ECONNABORTED); diff --git a/net/core/gso.c b/net/core/gso.c index bcd156372f4d..e96ef6350064 100644 --- a/net/core/gso.c +++ b/net/core/gso.c @@ -117,6 +117,7 @@ struct sk_buff *__skb_gso_segment(struct sk_buff *skb, SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb); SKB_GSO_CB(skb)->encap_level = 0; + SKB_GSO_CB(skb)->recursion_counter = 0; skb_reset_mac_header(skb); skb_reset_mac_len(skb); diff --git a/net/core/netdev-genl.c b/net/core/netdev-genl.c index cb18db681640..33b9f4eb9565 100644 --- a/net/core/netdev-genl.c +++ b/net/core/netdev-genl.c @@ -1168,6 +1168,12 @@ netdev_find_netmem_tx_dev(struct net_device *dev) return NULL; } +/* Note: NETDEV_CMD_BIND_TX is intentionally unprivileged (no + * GENL_ADMIN_PERM / GENL_UNS_ADMIN_PERM). Unlike bind-rx, which configures + * shared NIC RX queues, bind-tx only DMA-maps the caller's dmabuf so they can + * transmit from it on their own sockets without affecting other traffic or + * device state. + */ int netdev_nl_bind_tx_doit(struct sk_buff *skb, struct genl_info *info) { struct net_devmem_dmabuf_binding *binding; diff --git a/net/core/page_pool.c b/net/core/page_pool.c index 08d7f35cf608..d7c88c0b67e5 100644 --- a/net/core/page_pool.c +++ b/net/core/page_pool.c @@ -951,8 +951,8 @@ static void page_pool_recycle_ring_bulk(struct page_pool *pool, } } - page_pool_producer_unlock(pool, in_softirq); recycle_stat_add(pool, ring, i); + page_pool_producer_unlock(pool, in_softirq); /* Hopefully all pages were returned into ptr_ring */ if (likely(i == bulk_len)) diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 609f2c7f4a47..4aea06d5167d 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -5977,7 +5977,8 @@ static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate) err = skb_maybe_pull_tail(skb, off + sizeof(struct ipv6_opt_hdr), - MAX_IPV6_HDR_LEN); + off + + sizeof(struct ipv6_opt_hdr)); if (err < 0) goto out; @@ -5992,7 +5993,8 @@ static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate) err = skb_maybe_pull_tail(skb, off + sizeof(struct ip_auth_hdr), - MAX_IPV6_HDR_LEN); + off + + sizeof(struct ip_auth_hdr)); if (err < 0) goto out; @@ -6007,7 +6009,8 @@ static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate) err = skb_maybe_pull_tail(skb, off + sizeof(struct frag_hdr), - MAX_IPV6_HDR_LEN); + off + + sizeof(struct frag_hdr)); if (err < 0) goto out; @@ -7357,8 +7360,8 @@ void skb_attempt_defer_free(struct sk_buff *skb) struct skb_defer_node *sdn; unsigned long defer_count; unsigned int defer_max; + int cpu, my_cpu; bool kick; - int cpu; if (static_branch_unlikely(&skb_defer_disable_key)) goto nodefer; @@ -7368,7 +7371,8 @@ void skb_attempt_defer_free(struct sk_buff *skb) goto nodefer; cpu = skb->alloc_cpu; - if (cpu == raw_smp_processor_id() || + my_cpu = raw_smp_processor_id(); + if (cpu == my_cpu || WARN_ON_ONCE(cpu >= nr_cpu_ids) || !cpu_online(cpu)) { nodefer: kfree_skb_napi_cache(skb); @@ -7379,7 +7383,7 @@ nodefer: kfree_skb_napi_cache(skb); DEBUG_NET_WARN_ON_ONCE(skb->destructor); DEBUG_NET_WARN_ON_ONCE(skb_nfct(skb)); - sdn = per_cpu_ptr(net_hotdata.skb_defer_nodes, cpu) + numa_node_id(); + sdn = per_cpu_ptr(net_hotdata.skb_defer_nodes, cpu) + cpu_to_node(my_cpu); defer_max = READ_ONCE(net_hotdata.sysctl_skb_defer_max); defer_count = atomic_long_inc_return(&sdn->defer_count); @@ -7389,6 +7393,11 @@ nodefer: kfree_skb_napi_cache(skb); llist_add(&skb->ll_node, &sdn->defer_list); + if (unlikely(!cpu_online(cpu) || my_cpu != raw_smp_processor_id())) { + skb_defer_node_flush(sdn); + return; + } + /* Send an IPI every time queue reaches half capacity. */ kick = (defer_count - 1) == (defer_max >> 1); @@ -7400,7 +7409,8 @@ nodefer: kfree_skb_napi_cache(skb); } static void skb_splice_csum_page(struct sk_buff *skb, struct page *page, - size_t offset, size_t len) + size_t offset, size_t len, + unsigned int csum_offset) { const char *kaddr; __wsum csum; @@ -7408,7 +7418,7 @@ static void skb_splice_csum_page(struct sk_buff *skb, struct page *page, kaddr = kmap_local_page(page); csum = csum_partial(kaddr + offset, len, 0); kunmap_local(kaddr); - skb->csum = csum_block_add(skb->csum, csum, skb->len); + skb->csum = csum_block_add(skb->csum, csum, csum_offset); } /** @@ -7468,7 +7478,8 @@ ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter, } if (skb->ip_summed == CHECKSUM_NONE) - skb_splice_csum_page(skb, page, off, part); + skb_splice_csum_page(skb, page, off, part, + skb->len + spliced); off = 0; spliced += part; diff --git a/net/core/skmsg.c b/net/core/skmsg.c index 2521b643fa05..df385a5a961e 100644 --- a/net/core/skmsg.c +++ b/net/core/skmsg.c @@ -1000,6 +1000,10 @@ static int sk_psock_verdict_apply(struct sk_psock *psock, struct sk_buff *skb, int err = 0; u32 len, off; + if (verdict == __SK_REDIRECT && skb_bpf_ingress(skb) && + skb_bpf_redirect_fetch(skb) == psock->sk) + verdict = __SK_PASS; + switch (verdict) { case __SK_PASS: err = -EIO; diff --git a/net/core/sock.c b/net/core/sock.c index d23333bb4f3f..e8551df8330f 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -1034,7 +1034,7 @@ static int sock_reserve_memory(struct sock *sk, int bytes) bool charged; int pages; - if (!mem_cgroup_sk_enabled(sk) || !sk_has_account(sk)) + if (!mem_cgroup_sk_enabled(sk) || !sk_is_tcp(sk)) return -EOPNOTSUPP; if (!bytes) @@ -1661,7 +1661,7 @@ int sk_setsockopt(struct sock *sk, int level, int optname, { int delta; - if (val < 0) { + if (val < 0 || val > SZ_1G) { ret = -EINVAL; break; } diff --git a/net/core/sock_map.c b/net/core/sock_map.c index ca49bc7f8687..38df84284328 100644 --- a/net/core/sock_map.c +++ b/net/core/sock_map.c @@ -41,6 +41,7 @@ static struct bpf_map *sock_map_alloc(union bpf_attr *attr) struct bpf_stab *stab; if (attr->max_entries == 0 || + attr->max_entries > INT_MAX || attr->key_size != 4 || (attr->value_size != sizeof(u32) && attr->value_size != sizeof(u64)) || diff --git a/net/ethtool/common.h b/net/ethtool/common.h index 4e5356e26f40..ae32e7fdb563 100644 --- a/net/ethtool/common.h +++ b/net/ethtool/common.h @@ -163,6 +163,8 @@ ethtool_ioctl_needs_rtnl(const struct net_device *dev, u32 ethcmd) return ops->op_needs_rtnl & ETHTOOL_OP_NEEDS_RTNL_RSS; case ETHTOOL_GLINK: return ops->op_needs_rtnl & ETHTOOL_OP_NEEDS_RTNL_GLINK; + case ETHTOOL_TEST: + return ops->op_needs_rtnl & ETHTOOL_OP_NEEDS_RTNL_TEST; } return false; } diff --git a/net/ethtool/tsconfig.c b/net/ethtool/tsconfig.c index 6be3aa5d4bc1..2db0e7ba8b9f 100644 --- a/net/ethtool/tsconfig.c +++ b/net/ethtool/tsconfig.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only #include +#include #include #include @@ -42,7 +43,9 @@ static int tsconfig_prepare_data(const struct ethnl_req_info *req_base, struct kernel_hwtstamp_config cfg = {}; int ret; - if (!dev->netdev_ops->ndo_hwtstamp_get) + if (!dev->netdev_ops->ndo_hwtstamp_get && + !phy_is_default_hwtstamp(dev->phydev) && + !netdev_ops_lock_dereference(dev->hwprov, dev)) return -EOPNOTSUPP; ret = ethnl_ops_begin(dev); @@ -248,17 +251,6 @@ static int tsconfig_send_reply(struct net_device *dev, struct genl_info *info) return ret; } -static int ethnl_set_tsconfig_validate(struct ethnl_req_info *req_base, - struct genl_info *info) -{ - const struct net_device_ops *ops = req_base->dev->netdev_ops; - - if (!ops->ndo_hwtstamp_set || !ops->ndo_hwtstamp_get) - return -EOPNOTSUPP; - - return 1; -} - static struct hwtstamp_provider * tsconfig_set_hwprov_from_desc(struct net_device *dev, struct genl_info *info, @@ -272,7 +264,7 @@ tsconfig_set_hwprov_from_desc(struct net_device *dev, int ret; ret = ethtool_net_get_ts_info_by_phc(dev, &ts_info, hwprov_desc); - if (!ret) { + if (!ret && dev->netdev_ops->ndo_hwtstamp_set) { /* Found */ source = HWTSTAMP_SOURCE_NETDEV; } else { @@ -313,6 +305,11 @@ static int ethnl_set_tsconfig(struct ethnl_req_info *req_base, if (!netif_device_present(dev)) return -ENODEV; + if (!dev->netdev_ops->ndo_hwtstamp_set && + !phy_is_default_hwtstamp(dev->phydev) && + !netdev_ops_lock_dereference(dev->hwprov, dev)) + return -EOPNOTSUPP; + if (tb[ETHTOOL_A_TSCONFIG_HWTSTAMP_PROVIDER]) { struct hwtstamp_provider_desc __hwprov_desc = {.index = -1}; struct hwtstamp_provider *__hwprov; @@ -459,6 +456,5 @@ const struct ethnl_request_ops ethnl_tsconfig_request_ops = { .reply_size = tsconfig_reply_size, .fill_reply = tsconfig_fill_reply, - .set_validate = ethnl_set_tsconfig_validate, .set = ethnl_set_tsconfig, }; diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c index 32d006c1a8ee..4ce38c99fef9 100644 --- a/net/ipv4/af_inet.c +++ b/net/ipv4/af_inet.c @@ -1375,6 +1375,9 @@ struct sk_buff *inet_gso_segment(struct sk_buff *skb, int id; skb_reset_network_header(skb); + if (unlikely(gso_recursion_inc_test(skb, + IP_TUNNEL_RECURSION_LIMIT))) + goto out; nhoff = skb_network_header(skb) - skb_mac_header(skb); if (unlikely(!pskb_may_pull(skb, sizeof(*iph)))) goto out; diff --git a/net/ipv4/arp.c b/net/ipv4/arp.c index d409f606aec0..60009d92e071 100644 --- a/net/ipv4/arp.c +++ b/net/ipv4/arp.c @@ -1278,6 +1278,7 @@ int arp_ioctl(struct net *net, unsigned int cmd, void __user *arg) err = copy_from_user(&r, arg, sizeof(struct arpreq)); if (err) return -EFAULT; + r.arp_dev[IFNAMSIZ - 1] = '\0'; break; default: return -EINVAL; diff --git a/net/ipv4/devinet.c b/net/ipv4/devinet.c index a90be57c63be..5b6b11c943e4 100644 --- a/net/ipv4/devinet.c +++ b/net/ipv4/devinet.c @@ -2117,9 +2117,10 @@ static int inet_validate_link_af(const struct net_device *dev, return err; if (tb[IFLA_INET_CONF]) { - err = nla_parse_nested(nested_tb, IPV4_DEVCONF_MAX, - tb[IFLA_INET_CONF], inet_devconf_policy, - extack); + err = nla_parse(nested_tb, IPV4_DEVCONF_MAX, + nla_data(tb[IFLA_INET_CONF]), + nla_len(tb[IFLA_INET_CONF]), + inet_devconf_policy, extack); if (err < 0) return err; diff --git a/net/ipv4/fib_semantics.c b/net/ipv4/fib_semantics.c index 7a362f2e2c2b..5c9021ea3a79 100644 --- a/net/ipv4/fib_semantics.c +++ b/net/ipv4/fib_semantics.c @@ -2176,6 +2176,15 @@ static bool fib_good_nh(const struct fib_nh *nh) return !!(state & NUD_VALID); } +static __be32 fib_nh_saddr(struct net *net, const struct fib_info *fi, + struct fib_nh *nh, int genid) +{ + if (READ_ONCE(nh->nh_saddr_genid) == genid) + return READ_ONCE(nh->nh_saddr); + + return fib_info_update_nhc_saddr(net, &nh->nh_common, fi->fib_scope); +} + void fib_select_multipath(struct fib_result *res, int hash, const struct flowi4 *fl4) { @@ -2184,6 +2193,7 @@ void fib_select_multipath(struct fib_result *res, int hash, bool use_neigh; int score = -1; __be32 saddr; + int genid; if (unlikely(res->fi->nh)) { nexthop_path_fib_result(res, hash); @@ -2192,6 +2202,7 @@ void fib_select_multipath(struct fib_result *res, int hash, use_neigh = READ_ONCE(net->ipv4.sysctl_fib_multipath_use_neigh); saddr = fl4 ? fl4->saddr : 0; + genid = saddr ? atomic_read(&net->ipv4.dev_addr_genid) : 0; change_nexthops(fi) { int nh_upper_bound, nh_score = 0; @@ -2204,7 +2215,7 @@ void fib_select_multipath(struct fib_result *res, int hash, (use_neigh && !fib_good_nh(nexthop_nh))) continue; - if (saddr && nexthop_nh->nh_saddr == saddr) + if (saddr && fib_nh_saddr(net, fi, nexthop_nh, genid) == saddr) nh_score += 2; if (hash <= nh_upper_bound) nh_score++; diff --git a/net/ipv4/fou_core.c b/net/ipv4/fou_core.c index 5e867f1b5c1d..076fdca44f51 100644 --- a/net/ipv4/fou_core.c +++ b/net/ipv4/fou_core.c @@ -600,6 +600,10 @@ static int fou_create(struct net *net, struct fou_cfg *cfg, /* Initial for fou type */ switch (cfg->type) { case FOU_ENCAP_DIRECT: + if (!cfg->protocol) { + err = -EINVAL; + goto error; + } tunnel_cfg.encap_rcv = fou_udp_recv; tunnel_cfg.gro_receive = fou_gro_receive; tunnel_cfg.gro_complete = fou_gro_complete; diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c index 6257459bcee2..6a30f1138454 100644 --- a/net/ipv4/inet_connection_sock.c +++ b/net/ipv4/inet_connection_sock.c @@ -1520,7 +1520,8 @@ void inet_csk_listen_stop(struct sock *sk) local_bh_enable(); sock_put(child); - cond_resched(); + if (!has_current_bpf_ctx()) + cond_resched(); } if (queue->fastopenq.rskq_rst_head) { /* Free all the reqs queued in rskq_rst_head. */ diff --git a/net/ipv4/ip_gre.c b/net/ipv4/ip_gre.c index 82309efd417e..e4878e9aa636 100644 --- a/net/ipv4/ip_gre.c +++ b/net/ipv4/ip_gre.c @@ -1464,6 +1464,12 @@ static int ipgre_changelink(struct net_device *dev, struct nlattr *tb[], if (!rtnl_dev_link_net_capable(dev, t->net)) return -EPERM; + if (data && data[IFLA_GRE_COLLECT_METADATA] && !t->collect_md) { + NL_SET_ERR_MSG(extack, + "Enabling collect_md on an existing device is not supported"); + return -EOPNOTSUPP; + } + err = ipgre_newlink_encap_setup(dev, data); if (err) return err; @@ -1496,6 +1502,12 @@ static int erspan_changelink(struct net_device *dev, struct nlattr *tb[], if (!rtnl_dev_link_net_capable(dev, t->net)) return -EPERM; + if (data && data[IFLA_GRE_COLLECT_METADATA] && !t->collect_md) { + NL_SET_ERR_MSG(extack, + "Enabling collect_md on an existing device is not supported"); + return -EOPNOTSUPP; + } + err = ipgre_newlink_encap_setup(dev, data); if (err) return err; diff --git a/net/ipv4/ipconfig.c b/net/ipv4/ipconfig.c index 155db067eaec..1b8585404a41 100644 --- a/net/ipv4/ipconfig.c +++ b/net/ipv4/ipconfig.c @@ -676,6 +676,24 @@ static const u8 ic_bootp_cookie[4] = { 99, 130, 83, 99 }; #ifdef IPCONFIG_DHCP +static bool __init +ic_dhcp_add_option(u8 **options, const u8 *end, u8 type, const void *value, + int len) +{ + u8 *e = *options; + + /* leave room for the option header and the END marker */ + if (len > U8_MAX || end - e < len + 3) + return false; + + *e++ = type; + *e++ = len; + memcpy(e, value, len); + *options = e + len; + + return true; +} + static void __init ic_dhcp_init_options(u8 *options, struct ic_device *d) { @@ -691,6 +709,7 @@ ic_dhcp_init_options(u8 *options, struct ic_device *d) 42, /* NTP servers */ }; u8 mt = (ic_servaddr == NONE) ? DHCPDISCOVER : DHCPREQUEST; + u8 *end = options + sizeof(((struct bootp_pkt *)0)->exten); u8 *e = options; int len; @@ -721,31 +740,19 @@ ic_dhcp_init_options(u8 *options, struct ic_device *d) e += sizeof(ic_req_params); if (ic_host_name_set) { - *e++ = 12; /* host-name */ len = strlen(utsname()->nodename); - *e++ = len; - memcpy(e, utsname()->nodename, len); - e += len; + ic_dhcp_add_option(&e, end, 12, utsname()->nodename, len); } if (*vendor_class_identifier) { - pr_info("DHCP: sending class identifier \"%s\"\n", - vendor_class_identifier); - *e++ = 60; /* Class-identifier */ len = strlen(vendor_class_identifier); - *e++ = len; - memcpy(e, vendor_class_identifier, len); - e += len; + if (ic_dhcp_add_option(&e, end, 60, vendor_class_identifier, len)) + pr_info("DHCP: sending class identifier \"%s\"\n", + vendor_class_identifier); } len = strlen(dhcp_client_identifier + 1); - /* the minimum length of identifier is 2, include 1 byte type, - * and can not be larger than the length of options - */ - if (len >= 1 && len < 312 - (e - options) - 1) { - *e++ = 61; - *e++ = len + 1; - memcpy(e, dhcp_client_identifier, len + 1); - e += len + 1; - } + /* the minimum length of identifier is 2, include 1 byte type */ + if (len >= 1) + ic_dhcp_add_option(&e, end, 61, dhcp_client_identifier, len + 1); *e++ = 255; /* End of the list */ } diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index 92bc60716f33..99baf14afdfd 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -4465,6 +4465,12 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) return 1; old_ack: + /* An old ACK can carry new data. Update TS.Recent before SACK + * processing can trigger a retransmission. + */ + if (flag & FLAG_UPDATE_TS_RECENT) + tcp_replace_ts_recent(tp, TCP_SKB_CB(skb)->seq); + /* If data was SACKed, tag it and see if we should send more data. * If data was DSACKed, see if we can undo a cwnd reduction. */ diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index d960e3de7d50..e0c392e29de5 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -3886,6 +3886,7 @@ void tcp_send_active_reset(struct sock *sk, enum sk_rst_reason reason) */ int tcp_send_synack(struct sock *sk) { + struct tcp_sock *tp = tcp_sk(sk); struct sk_buff *skb; skb = tcp_rtx_queue_head(sk); @@ -3903,6 +3904,8 @@ int tcp_send_synack(struct sock *sk) if (!nskb) return -ENOMEM; INIT_LIST_HEAD(&nskb->tcp_tsorted_anchor); + if (skb == tp->retransmit_skb_hint) + tp->retransmit_skb_hint = nskb; tcp_highest_sack_replace(sk, skb, nskb); tcp_rtx_queue_unlink_and_free(skb, sk); __skb_header_release(nskb); diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index bb8cfc62cb00..b090bd1f59e8 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -617,14 +617,23 @@ void udp_lib_hash4(struct sock *sk, u16 hash) struct net *net = sock_net(sk); struct udp_table *udptable; - /* Connected udp socket can re-connect to another remote address, which - * will be handled by rehash. Thus no need to redo hash4 here. - */ - if (udp_hashed4(sk)) - return; - udptable = net->ipv4.udp_table; hslot = udp_hashslot(udptable, net, udp_sk(sk)->udp_port_hash); + + /* A connected socket can re-connect to another address. rehash() + * relocates it, but only runs when the local address changes, so a + * socket bound to a specific address would stay filed under the + * previous peer's hash. Move it here. + */ + if (udp_hashed4(sk)) { + if (udp_sk(sk)->udp_lrpa_hash != hash) { + spin_lock_bh(&hslot->lock); + udp_rehash4(udptable, sk, hash); + spin_unlock_bh(&hslot->lock); + } + return; + } + hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash); hslot4 = udp_hashslot4(udptable, hash); udp_sk(sk)->udp_lrpa_hash = hash; @@ -2197,9 +2206,31 @@ int __udp_disconnect(struct sock *sk, int flags) } EXPORT_SYMBOL(__udp_disconnect); +/* __udp_disconnect() takes a socket out of the 4-tuple hash table only via + * ->rehash() or ->unhash(), and neither runs for a socket bound to a + * specific address and port. Remove it here, before its peer is cleared. + */ +static void udp_unhash4_on_disconnect(struct sock *sk) +{ + struct net *net = sock_net(sk); + struct udp_table *udptable; + struct udp_hslot *hslot; + + if (!udp_hashed4(sk)) + return; + + udptable = net->ipv4.udp_table; + hslot = udp_hashslot(udptable, net, udp_sk(sk)->udp_port_hash); + + spin_lock_bh(&hslot->lock); + udp_unhash4(udptable, sk); + spin_unlock_bh(&hslot->lock); +} + int udp_disconnect(struct sock *sk, int flags) { lock_sock(sk); + udp_unhash4_on_disconnect(sk); __udp_disconnect(sk, flags); release_sock(sk); return 0; @@ -3131,6 +3162,7 @@ int udp_abort(struct sock *sk, int err) sk->sk_err = err; sk_error_report(sk); + udp_unhash4_on_disconnect(sk); __udp_disconnect(sk, 0); out: diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index 9d89be7e0544..c90ee6dd7446 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -4874,7 +4874,7 @@ static int modify_prefix_route(struct net *net, struct inet6_ifaddr *ifp, fib6_clean_expires(f6i); fib6_may_remove_gc_list(net, f6i); } else { - fib6_set_expires(f6i, expires); + fib6_set_expires(f6i, jiffies + clock_t_to_jiffies(expires)); fib6_add_gc_list(f6i); } diff --git a/net/ipv6/exthdrs_core.c b/net/ipv6/exthdrs_core.c index 9d06d487e8b1..4a9748338cf4 100644 --- a/net/ipv6/exthdrs_core.c +++ b/net/ipv6/exthdrs_core.c @@ -278,6 +278,9 @@ int ipv6_find_hdr(const struct sk_buff *skb, unsigned int *offset, hdrlen = ipv6_optlen(hp); if (!found) { + if (skb->len - start < hdrlen) + return -EBADMSG; + nexthdr = hp->nexthdr; start += hdrlen; } diff --git a/net/ipv6/ip6_fib.c b/net/ipv6/ip6_fib.c index 9ea75703b38d..9ff761962b45 100644 --- a/net/ipv6/ip6_fib.c +++ b/net/ipv6/ip6_fib.c @@ -1043,8 +1043,8 @@ static void fib6_purge_rt(struct fib6_info *rt, struct fib6_node *fn, struct fib6_table *table = rt->fib6_table; /* Flush all cached dst in exception table */ - rt6_flush_exceptions(rt); fib6_drop_pcpu_from(rt); + rt6_flush_exceptions(rt); if (rt->nh) { spin_lock(&rt->nh->lock); diff --git a/net/ipv6/ip6_gre.c b/net/ipv6/ip6_gre.c index 8ebda0b6a78b..e61cb10b50dc 100644 --- a/net/ipv6/ip6_gre.c +++ b/net/ipv6/ip6_gre.c @@ -2279,7 +2279,7 @@ static int ip6erspan_changelink(struct net_device *dev, struct nlattr *tb[], return PTR_ERR(t); ip6erspan_set_version(data, &p); - ip6gre_tunnel_unlink_md(ign, t); + ip6erspan_tunnel_unlink_md(ign, t); ip6gre_tunnel_unlink(ign, t); ip6erspan_tnl_change(t, &p, !tb[IFLA_MTU]); ip6erspan_tunnel_link_md(ign, t); diff --git a/net/ipv6/ip6_offload.c b/net/ipv6/ip6_offload.c index 78f50c93c536..700f1861d8eb 100644 --- a/net/ipv6/ip6_offload.c +++ b/net/ipv6/ip6_offload.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "ip6_offload.h" #include "tcpv6_offload.c" @@ -104,6 +105,9 @@ static struct sk_buff *ipv6_gso_segment(struct sk_buff *skb, bool gso_partial; skb_reset_network_header(skb); + if (unlikely(gso_recursion_inc_test(skb, + IP_TUNNEL_RECURSION_LIMIT))) + goto out; nhoff = skb_network_header(skb) - skb_mac_header(skb); if (unlikely(!pskb_may_pull(skb, sizeof(*ipv6h)))) goto out; diff --git a/net/ipv6/netfilter/ip6t_rpfilter.c b/net/ipv6/netfilter/ip6t_rpfilter.c index 67c87a88cde4..b5def30c3127 100644 --- a/net/ipv6/netfilter/ip6t_rpfilter.c +++ b/net/ipv6/netfilter/ip6t_rpfilter.c @@ -61,7 +61,7 @@ static bool rpfilter_lookup_reverse6(struct net *net, const struct sk_buff *skb, fl6.flowi6_oif = dev->ifindex; rt = (void *)ip6_route_lookup(net, &fl6, skb, lookup_flags); - if (rt->dst.error) + if (rt->dst.error || !rt->rt6i_idev) goto out; if (rt->rt6i_flags & (RTF_REJECT|RTF_ANYCAST)) diff --git a/net/ipv6/netfilter/ip6t_rt.c b/net/ipv6/netfilter/ip6t_rt.c index 8051425213dd..9880faf3cc7d 100644 --- a/net/ipv6/netfilter/ip6t_rt.c +++ b/net/ipv6/netfilter/ip6t_rt.c @@ -96,7 +96,8 @@ static bool rt_mt6(const struct sk_buff *skb, struct xt_action_param *par) unsigned int i = 0; for (temp = 0; - temp < (unsigned int)((hdrlen - 8) / 16); + temp < (unsigned int)((hdrlen - 8) / 16) && + i < rtinfo->addrnr; temp++) { ap = skb_header_pointer(skb, ptr @@ -112,8 +113,6 @@ static bool rt_mt6(const struct sk_buff *skb, struct xt_action_param *par) if (ipv6_addr_equal(ap, &rtinfo->addrs[i])) i++; - if (i == rtinfo->addrnr) - break; } if (i == rtinfo->addrnr) return ret; @@ -162,6 +161,12 @@ static int rt_mt6_check(const struct xt_mtchk_param *par) pr_info_ratelimited("too many addresses specified\n"); return -EINVAL; } + + if ((rtinfo->flags & IP6T_RT_FST_MASK) && !rtinfo->addrnr) { + pr_info_ratelimited("address list match requested but addrnr is 0\n"); + return -EINVAL; + } + if ((rtinfo->flags & (IP6T_RT_RES | IP6T_RT_FST_MASK)) && (!(rtinfo->flags & IP6T_RT_TYP) || (rtinfo->rt_type != 0) || diff --git a/net/ipv6/route.c b/net/ipv6/route.c index 08bd68f1b5bb..153ce16628c1 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -139,6 +139,7 @@ void rt6_uncached_list_add(struct rt6_info *rt) { struct uncached_list *ul = raw_cpu_ptr(&rt6_uncached_list); + /* Set once and never cleared: non-NULL marks an uncached route. */ rt->dst.rt_uncached_list = ul; spin_lock_bh(&ul->lock); @@ -1729,6 +1730,11 @@ static int rt6_insert_exception(struct rt6_info *nrt, spin_lock_bh(&rt6_exception_lock); + if (f6i->fib6_destroying) { + err = -ENOENT; + goto out; + } + bucket = rcu_dereference_protected(nh->rt6i_exception_bucket, lockdep_is_held(&rt6_exception_lock)); if (!bucket) { @@ -2721,8 +2727,8 @@ struct dst_entry *ip6_route_output_flags(struct net *net, rcu_read_lock(); dst = ip6_route_output_flags_noref(net, sk, fl6, flags); rt6 = dst_rt6_info(dst); - /* For dst cached in uncached_list, refcnt is already taken. */ - if (list_empty(&rt6->dst.rt_uncached) && !dst_hold_safe(dst)) { + /* For an uncached dst, refcnt is already taken. */ + if (!rt6->dst.rt_uncached_list && !dst_hold_safe(dst)) { dst = &net->ipv6.ip6_null_entry->dst; dst_hold(dst); } @@ -2831,7 +2837,7 @@ INDIRECT_CALLABLE_SCOPE struct dst_entry *ip6_dst_check(struct dst_entry *dst, from = rcu_dereference(rt->from); if (from && (rt->rt6i_flags & RTF_PCPU || - unlikely(!list_empty(&rt->dst.rt_uncached)))) + unlikely(rt->dst.rt_uncached_list))) dst_ret = rt6_dst_from_check(rt, from, cookie); else dst_ret = rt6_check(rt, from, cookie); diff --git a/net/ipv6/seg6.c b/net/ipv6/seg6.c index 62a7eb779202..8c2b156c227a 100644 --- a/net/ipv6/seg6.c +++ b/net/ipv6/seg6.c @@ -138,8 +138,8 @@ void seg6_icmp_srh(struct sk_buff *skb, struct inet6_skb_parm *opt) static struct genl_family seg6_genl_family; static const struct nla_policy seg6_genl_policy[SEG6_ATTR_MAX + 1] = { - [SEG6_ATTR_DST] = { .type = NLA_BINARY, - .len = sizeof(struct in6_addr) }, + [SEG6_ATTR_DST] = + NLA_POLICY_EXACT_LEN(sizeof(struct in6_addr)), [SEG6_ATTR_DSTLEN] = { .type = NLA_S32, }, [SEG6_ATTR_HMACKEYID] = { .type = NLA_U32, }, [SEG6_ATTR_SECRET] = { .type = NLA_BINARY, }, diff --git a/net/ipv6/seg6_iptunnel.c b/net/ipv6/seg6_iptunnel.c index 61c6a27bf202..7cbe0685f8bb 100644 --- a/net/ipv6/seg6_iptunnel.c +++ b/net/ipv6/seg6_iptunnel.c @@ -129,7 +129,7 @@ static __be32 seg6_make_flowlabel(struct net *net, struct sk_buff *skb, u32 hash; if (do_flowlabel > 0) { - hash = skb_get_hash(skb); + hash = skb_get_hash_net(net, skb); hash = rol32(hash, 16); flowlabel = (__force __be32)hash & IPV6_FLOWLABEL_MASK; } else if (!do_flowlabel && skb->protocol == htons(ETH_P_IPV6)) { diff --git a/net/ipv6/seg6_local.c b/net/ipv6/seg6_local.c index d1070aec7b72..53f7b2452575 100644 --- a/net/ipv6/seg6_local.c +++ b/net/ipv6/seg6_local.c @@ -278,13 +278,43 @@ static bool decap_and_validate(struct sk_buff *skb, int proto) return true; } -static void advance_nextseg(struct ipv6_sr_hdr *srh, struct in6_addr *daddr) +/* advance the SRH to the next segment and set the IPv6 DA accordingly. + * skb pointers may change: after this call, the caller must evaluate again + * any pointer into the packet data. + * + * This function returns: + * - the SRH on success; + * - NULL when the skb cannot be made writable. In this case, the function + * sets the drop reason to SKB_DROP_REASON_NOMEM. + */ +static struct ipv6_sr_hdr *advance_nextseg(struct sk_buff *skb, + struct ipv6_sr_hdr *srh, + enum skb_drop_reason *reason) { struct in6_addr *addr; + int srhoff; + int wlen; + + srhoff = (unsigned char *)srh - skb->data; + /* we write only the Segment Left field and the IPv6 DA. The segment + * list is only read, and seg6_get_srh() already pulled it into the + * linear area. + */ + wlen = srhoff + sizeof(*srh); + + if (unlikely(skb_ensure_writable(skb, wlen))) { + *reason = SKB_DROP_REASON_NOMEM; + return NULL; + } + + /* skb_ensure_writable() may change skb pointers; evaluate srh again */ + srh = (struct ipv6_sr_hdr *)(skb->data + srhoff); srh->segments_left--; addr = srh->segments + srh->segments_left; - *daddr = *addr; + ipv6_hdr(skb)->daddr = *addr; + + return srh; } static int @@ -382,12 +412,26 @@ static bool seg6_next_csid_is_arg_zero(const struct in6_addr *addr, return true; } -/* assume that DA.Argument length > 0 */ -static void seg6_next_csid_advance_arg(struct in6_addr *addr, - const struct seg6_flavors_info *finfo) +/* assume that DA.Argument length > 0. + * skb pointers may change: after this call, the caller must evaluate again + * any pointer into the packet data. + * + * This function returns: + * - SKB_NOT_DROPPED_YET on success; + * - SKB_DROP_REASON_NOMEM when the skb cannot be made writable. + */ +static enum skb_drop_reason +seg6_next_csid_advance_arg(struct sk_buff *skb, + const struct seg6_flavors_info *finfo) { __u8 fnc_octects = seg6_flv_lcnode_func_octects(finfo); __u8 blk_octects = seg6_flv_lcblock_octects(finfo); + struct in6_addr *addr; + + if (unlikely(skb_ensure_writable(skb, sizeof(struct ipv6hdr)))) + return SKB_DROP_REASON_NOMEM; + + addr = &ipv6_hdr(skb)->daddr; /* advance DA.Argument */ memmove(&addr->s6_addr[blk_octects], @@ -395,6 +439,8 @@ static void seg6_next_csid_advance_arg(struct in6_addr *addr, 16 - blk_octects - fnc_octects); memset(&addr->s6_addr[16 - fnc_octects], 0x00, fnc_octects); + + return SKB_NOT_DROPPED_YET; } static int input_action_end_finish(struct sk_buff *skb, @@ -408,31 +454,42 @@ static int input_action_end_finish(struct sk_buff *skb, static int input_action_end_core(struct sk_buff *skb, struct seg6_local_lwt *slwt) { + enum skb_drop_reason reason = SKB_DROP_REASON_NOT_SPECIFIED; struct ipv6_sr_hdr *srh; + int err = -EINVAL; srh = get_and_validate_srh(skb); if (!srh) goto drop; - advance_nextseg(srh, &ipv6_hdr(skb)->daddr); + srh = advance_nextseg(skb, srh, &reason); + if (!srh) { + err = -ENOMEM; + goto drop; + } return input_action_end_finish(skb, slwt); drop: - kfree_skb(skb); - return -EINVAL; + kfree_skb_reason(skb, reason); + return err; } static int end_next_csid_core(struct sk_buff *skb, struct seg6_local_lwt *slwt) { const struct seg6_flavors_info *finfo = &slwt->flv_info; struct in6_addr *daddr = &ipv6_hdr(skb)->daddr; + enum skb_drop_reason reason; if (seg6_next_csid_is_arg_zero(daddr, finfo)) return input_action_end_core(skb, slwt); /* update DA */ - seg6_next_csid_advance_arg(daddr, finfo); + reason = seg6_next_csid_advance_arg(skb, finfo); + if (reason) { + kfree_skb_reason(skb, reason); + return -ENOMEM; + } return input_action_end_finish(skb, slwt); } @@ -448,19 +505,25 @@ static int input_action_end_x_finish(struct sk_buff *skb, static int input_action_end_x_core(struct sk_buff *skb, struct seg6_local_lwt *slwt) { + enum skb_drop_reason reason = SKB_DROP_REASON_NOT_SPECIFIED; struct ipv6_sr_hdr *srh; + int err = -EINVAL; srh = get_and_validate_srh(skb); if (!srh) goto drop; - advance_nextseg(srh, &ipv6_hdr(skb)->daddr); + srh = advance_nextseg(skb, srh, &reason); + if (!srh) { + err = -ENOMEM; + goto drop; + } return input_action_end_x_finish(skb, slwt); drop: - kfree_skb(skb); - return -EINVAL; + kfree_skb_reason(skb, reason); + return err; } static int end_x_next_csid_core(struct sk_buff *skb, @@ -468,12 +531,17 @@ static int end_x_next_csid_core(struct sk_buff *skb, { const struct seg6_flavors_info *finfo = &slwt->flv_info; struct in6_addr *daddr = &ipv6_hdr(skb)->daddr; + enum skb_drop_reason reason; if (seg6_next_csid_is_arg_zero(daddr, finfo)) return input_action_end_x_core(skb, slwt); /* update DA */ - seg6_next_csid_advance_arg(daddr, finfo); + reason = seg6_next_csid_advance_arg(skb, finfo); + if (reason) { + kfree_skb_reason(skb, reason); + return -ENOMEM; + } return input_action_end_x_finish(skb, slwt); } @@ -760,10 +828,12 @@ static bool seg6_pop_srh(struct sk_buff *skb, int srhoff) */ static int end_flv8986_core(struct sk_buff *skb, struct seg6_local_lwt *slwt) { + enum skb_drop_reason reason = SKB_DROP_REASON_NOT_SPECIFIED; const struct seg6_flavors_info *finfo = &slwt->flv_info; enum seg6_local_flv_action action; enum seg6_local_pktinfo pinfo; struct ipv6_sr_hdr *srh; + int err = -EINVAL; __u32 flvmask; int srhoff; @@ -787,10 +857,18 @@ static int end_flv8986_core(struct sk_buff *skb, struct seg6_local_lwt *slwt) switch (action) { case SEG6_LOCAL_FLV_ACT_END: /* process the packet as the "standard" End behavior */ - advance_nextseg(srh, &ipv6_hdr(skb)->daddr); + srh = advance_nextseg(skb, srh, &reason); + if (!srh) { + err = -ENOMEM; + goto drop; + } break; case SEG6_LOCAL_FLV_ACT_PSP: - advance_nextseg(srh, &ipv6_hdr(skb)->daddr); + srh = advance_nextseg(skb, srh, &reason); + if (!srh) { + err = -ENOMEM; + goto drop; + } if (unlikely(!seg6_pop_srh(skb, srhoff))) goto drop; @@ -807,8 +885,8 @@ static int end_flv8986_core(struct sk_buff *skb, struct seg6_local_lwt *slwt) return input_action_end_finish(skb, slwt); drop: - kfree_skb(skb); - return -EINVAL; + kfree_skb_reason(skb, reason); + return err; } /* regular endpoint function */ @@ -847,21 +925,27 @@ static int input_action_end_x(struct sk_buff *skb, struct seg6_local_lwt *slwt) static int input_action_end_t(struct sk_buff *skb, struct seg6_local_lwt *slwt) { + enum skb_drop_reason reason = SKB_DROP_REASON_NOT_SPECIFIED; struct ipv6_sr_hdr *srh; + int err = -EINVAL; srh = get_and_validate_srh(skb); if (!srh) goto drop; - advance_nextseg(srh, &ipv6_hdr(skb)->daddr); + srh = advance_nextseg(skb, srh, &reason); + if (!srh) { + err = -ENOMEM; + goto drop; + } seg6_lookup_nexthop(skb, NULL, slwt->table); return dst_input(skb); drop: - kfree_skb(skb); - return -EINVAL; + kfree_skb_reason(skb, reason); + return err; } /* decapsulate and forward inner L2 frame on specified interface */ @@ -1375,6 +1459,7 @@ static int input_action_end_b6(struct sk_buff *skb, struct seg6_local_lwt *slwt) static int input_action_end_b6_encap(struct sk_buff *skb, struct seg6_local_lwt *slwt) { + enum skb_drop_reason reason = SKB_DROP_REASON_NOT_SPECIFIED; struct ipv6_sr_hdr *srh; int err = -EINVAL; @@ -1382,7 +1467,11 @@ static int input_action_end_b6_encap(struct sk_buff *skb, if (!srh) goto drop; - advance_nextseg(srh, &ipv6_hdr(skb)->daddr); + srh = advance_nextseg(skb, srh, &reason); + if (!srh) { + err = -ENOMEM; + goto drop; + } skb_reset_inner_headers(skb); skb->encapsulation = 1; @@ -1398,7 +1487,7 @@ static int input_action_end_b6_encap(struct sk_buff *skb, return dst_input(skb); drop: - kfree_skb(skb); + kfree_skb_reason(skb, reason); return err; } @@ -1434,6 +1523,7 @@ static int input_action_end_bpf(struct sk_buff *skb, struct seg6_local_lwt *slwt) { struct seg6_bpf_srh_state *srh_state; + enum skb_drop_reason reason; struct ipv6_sr_hdr *srh; int ret; @@ -1442,7 +1532,11 @@ static int input_action_end_bpf(struct sk_buff *skb, kfree_skb(skb); return -EINVAL; } - advance_nextseg(srh, &ipv6_hdr(skb)->daddr); + srh = advance_nextseg(skb, srh, &reason); + if (!srh) { + kfree_skb_reason(skb, reason); + return -ENOMEM; + } /* The access to the per-CPU buffer srh_state is protected by running * always in softirq context (with disabled BH). On PREEMPT_RT the diff --git a/net/llc/llc_c_ac.c b/net/llc/llc_c_ac.c index 724ecd741d4c..1aa7fe28acdd 100644 --- a/net/llc/llc_c_ac.c +++ b/net/llc/llc_c_ac.c @@ -437,7 +437,7 @@ int llc_conn_ac_resend_i_xxx_x_set_0_or_send_rr(struct sock *sk, if (likely(!rc)) llc_conn_send_pdu(sk, nskb); else - kfree_skb(skb); + kfree_skb(nskb); } if (rc) { nr = LLC_I_GET_NR(pdu); diff --git a/net/llc/llc_s_ac.c b/net/llc/llc_s_ac.c index 98deee560373..831998211b52 100644 --- a/net/llc/llc_s_ac.c +++ b/net/llc/llc_s_ac.c @@ -121,6 +121,8 @@ int llc_sap_action_send_xid_r(struct llc_sap *sap, struct sk_buff *skb) rc = llc_mac_hdr_init(nskb, mac_sa, mac_da); if (likely(!rc)) rc = dev_queue_xmit(nskb); + else + kfree_skb(nskb); out: return rc; } @@ -170,6 +172,8 @@ int llc_sap_action_send_test_r(struct llc_sap *sap, struct sk_buff *skb) rc = llc_mac_hdr_init(nskb, mac_sa, mac_da); if (likely(!rc)) rc = dev_queue_xmit(nskb); + else + kfree_skb(nskb); out: return rc; } diff --git a/net/llc/llc_sap.c b/net/llc/llc_sap.c index 1bd446a21092..3904a1b4ba84 100644 --- a/net/llc/llc_sap.c +++ b/net/llc/llc_sap.c @@ -19,12 +19,12 @@ #include #include -static int llc_mac_header_len(unsigned short devtype) +static int llc_mac_header_len(struct net_device *dev) { - switch (devtype) { + switch (dev->type) { case ARPHRD_ETHER: case ARPHRD_LOOPBACK: - return sizeof(struct ethhdr); + return LL_RESERVED_SPACE(dev); } return 0; } @@ -45,7 +45,7 @@ struct sk_buff *llc_alloc_frame(struct sock *sk, struct net_device *dev, int hlen = type == LLC_PDU_TYPE_U ? 3 : 4; struct sk_buff *skb; - hlen += llc_mac_header_len(dev->type); + hlen += llc_mac_header_len(dev); skb = alloc_skb(hlen + data_size, GFP_ATOMIC); if (skb) { diff --git a/net/mac80211/agg-rx.c b/net/mac80211/agg-rx.c index 9629e00069a1..01d6703767c6 100644 --- a/net/mac80211/agg-rx.c +++ b/net/mac80211/agg-rx.c @@ -102,7 +102,7 @@ void __ieee80211_stop_rx_ba_session(struct sta_info *sta, u16 tid, if (!tid_rx) return; - timer_delete_sync(&tid_rx->session_timer); + timer_shutdown_sync(&tid_rx->session_timer); /* make sure ieee80211_sta_reorder_release() doesn't re-arm the timer */ spin_lock_bh(&tid_rx->reorder_lock); diff --git a/net/mac80211/chan.c b/net/mac80211/chan.c index 75bb204ad743..9cba79507767 100644 --- a/net/mac80211/chan.c +++ b/net/mac80211/chan.c @@ -231,6 +231,13 @@ int ieee80211_chanctx_refcount(struct ieee80211_local *local, return num; } +static bool +ieee80211_chanctx_has_replace_partner(struct ieee80211_chanctx *ctx) +{ + return ctx->replace_state == IEEE80211_CHANCTX_WILL_BE_REPLACED && + ctx->replace_ctx; +} + static int ieee80211_num_chanctx(struct ieee80211_local *local, int radio_idx) { struct ieee80211_chanctx *ctx; @@ -1405,6 +1412,7 @@ void ieee80211_link_unreserve_chanctx(struct ieee80211_link_data *link) { struct ieee80211_sub_if_data *sdata = link->sdata; struct ieee80211_chanctx *ctx = link->reserved_chanctx; + struct ieee80211_chanctx *old_ctx = NULL; lockdep_assert_wiphy(sdata->local->hw.wiphy); @@ -1418,6 +1426,7 @@ void ieee80211_link_unreserve_chanctx(struct ieee80211_link_data *link) if (WARN_ON(!ctx->replace_ctx)) return; + old_ctx = ctx->replace_ctx; WARN_ON(ctx->replace_ctx->replace_state != IEEE80211_CHANCTX_WILL_BE_REPLACED); WARN_ON(ctx->replace_ctx->replace_ctx != ctx); @@ -1428,6 +1437,11 @@ void ieee80211_link_unreserve_chanctx(struct ieee80211_link_data *link) list_del_rcu(&ctx->list); kfree_rcu(ctx, rcu_head); + + if (ieee80211_chanctx_refcount(sdata->local, + old_ctx) == 0) + ieee80211_free_chanctx(sdata->local, old_ctx, + false); } else { ieee80211_free_chanctx(sdata->local, ctx, false); } @@ -1707,7 +1721,8 @@ ieee80211_link_use_reserved_reassign(struct ieee80211_link_data *link) ieee80211_check_fast_xmit_iface(sdata); - if (ieee80211_chanctx_refcount(local, old_ctx) == 0) + if (ieee80211_chanctx_refcount(local, old_ctx) == 0 && + !ieee80211_chanctx_has_replace_partner(old_ctx)) ieee80211_free_chanctx(local, old_ctx, false); ieee80211_recalc_chanctx_min_def(local, new_ctx); @@ -1772,7 +1787,8 @@ ieee80211_link_use_reserved_assign(struct ieee80211_link_data *link) } static bool -ieee80211_link_has_in_place_reservation(struct ieee80211_link_data *link) +ieee80211_link_has_in_place_reservation(struct ieee80211_link_data *link, + struct ieee80211_chanctx *ctx) { struct ieee80211_sub_if_data *sdata = link->sdata; struct ieee80211_chanctx *old_ctx, *new_ctx; @@ -1782,6 +1798,9 @@ ieee80211_link_has_in_place_reservation(struct ieee80211_link_data *link) new_ctx = link->reserved_chanctx; old_ctx = ieee80211_link_get_chanctx(link); + if (new_ctx != ctx) + return false; + if (!old_ctx) return false; @@ -1794,6 +1813,12 @@ ieee80211_link_has_in_place_reservation(struct ieee80211_link_data *link) if (new_ctx->replace_state != IEEE80211_CHANCTX_REPLACES_OTHER) return false; + if (new_ctx->replace_ctx != old_ctx) + return false; + + if (old_ctx->replace_ctx != new_ctx) + return false; + return true; } @@ -1823,7 +1848,8 @@ static int ieee80211_chsw_switch_vifs(struct ieee80211_local *local, } for_each_chanctx_user_reserved(local, ctx, &iter) { - if (!ieee80211_link_has_in_place_reservation(iter.link)) + if (!ieee80211_link_has_in_place_reservation(iter.link, + ctx)) continue; old_ctx = ieee80211_link_get_chanctx(iter.link); @@ -1925,7 +1951,9 @@ static int ieee80211_vif_use_reserved_switch(struct ieee80211_local *local) for_each_chanctx_user_assigned(local, ctx->replace_ctx, &iter) { n_assigned++; - if (iter.link && iter.link->reserved_chanctx) { + if (iter.link && + ieee80211_link_has_in_place_reservation(iter.link, + ctx)) { n_reserved++; if (iter.link->reserved_ready) n_ready++; @@ -1950,7 +1978,8 @@ static int ieee80211_vif_use_reserved_switch(struct ieee80211_local *local) use_reserved: ctx->conf.radar_enabled = false; for_each_chanctx_user_reserved(local, ctx, &iter) { - if (ieee80211_link_has_in_place_reservation(iter.link) && + if (ieee80211_link_has_in_place_reservation(iter.link, + ctx) && !iter.link->reserved_ready) return -EAGAIN; @@ -1991,7 +2020,8 @@ static int ieee80211_vif_use_reserved_switch(struct ieee80211_local *local) } for_each_chanctx_user_reserved(local, ctx, &iter) { - if (!ieee80211_link_has_in_place_reservation(iter.link)) + if (!ieee80211_link_has_in_place_reservation(iter.link, + ctx)) continue; ieee80211_chan_bw_change(local, @@ -2040,7 +2070,7 @@ static int ieee80211_vif_use_reserved_switch(struct ieee80211_local *local) struct ieee80211_bss_conf *link_conf = link->conf; u64 changed = 0; - if (!ieee80211_link_has_in_place_reservation(link)) + if (!ieee80211_link_has_in_place_reservation(link, ctx)) continue; rcu_assign_pointer(link_conf->chanctx_conf, @@ -2091,7 +2121,8 @@ static int ieee80211_vif_use_reserved_switch(struct ieee80211_local *local) for_each_chanctx_user_reserved(local, ctx, &iter) { struct ieee80211_link_data *link = iter.link; - if (WARN_ON(ieee80211_link_has_in_place_reservation(link))) + if (WARN_ON(ieee80211_link_has_in_place_reservation(link, + ctx))) continue; if (!link->reserved_ready) @@ -2176,7 +2207,8 @@ void __ieee80211_link_release_channel(struct ieee80211_link_data *link, } ieee80211_assign_link_chanctx(link, NULL, false); - if (ieee80211_chanctx_refcount(local, ctx) == 0) + if (ieee80211_chanctx_refcount(local, ctx) == 0 && + !ieee80211_chanctx_has_replace_partner(ctx)) ieee80211_free_chanctx(local, ctx, skip_idle_recalc); link->radar_required = false; diff --git a/net/mac80211/drop.h b/net/mac80211/drop.h index f06a8aa905c5..f433ee17cce5 100644 --- a/net/mac80211/drop.h +++ b/net/mac80211/drop.h @@ -108,6 +108,7 @@ typedef unsigned int __bitwise ieee80211_rx_result; R(RX_DROP_U_UNHANDLED_PREQ) \ R(RX_DROP_U_UNHANDLED_MGMT_STYPE) \ R(RX_DROP_U_NO_LINK) \ + R(RX_DROP_U_DEFRAG_OVERFLOW) \ /* this line for the trailing \ - add before this */ /* having two enums allows for checking ieee80211_rx_result use with sparse */ diff --git a/net/mac80211/fils_aead.c b/net/mac80211/fils_aead.c index 293590976489..9c4f00d61981 100644 --- a/net/mac80211/fils_aead.c +++ b/net/mac80211/fils_aead.c @@ -89,6 +89,14 @@ static int aes_siv_encrypt(const u8 *key, size_t key_len, if (res) return res; + /* With an empty plaintext there is no ciphertext to produce and + * the S2V result is the complete output. + */ + if (!plain_len) { + memcpy(out, v, AES_BLOCK_SIZE); + return 0; + } + /* Use a temporary buffer of the plaintext to handle need for * overwriting this during AES-CTR. */ diff --git a/net/mac80211/ieee80211_i.h b/net/mac80211/ieee80211_i.h index d05f59467399..9374c2942489 100644 --- a/net/mac80211/ieee80211_i.h +++ b/net/mac80211/ieee80211_i.h @@ -2475,7 +2475,7 @@ static inline bool ieee80211_require_encrypted_assoc(__le16 fc, /* sta_out needs to be checked for ERR_PTR() before using */ int ieee80211_lookup_ra_sta(struct ieee80211_sub_if_data *sdata, struct sk_buff *skb, - struct sta_info **sta_out); + struct sta_info **sta_out, bool bss); static inline void ieee80211_tx_skb_tid_band(struct ieee80211_sub_if_data *sdata, diff --git a/net/mac80211/iface.c b/net/mac80211/iface.c index 889c32fd8de1..5f0c482bd4a5 100644 --- a/net/mac80211/iface.c +++ b/net/mac80211/iface.c @@ -998,8 +998,8 @@ static u16 ieee80211_monitor_select_queue(struct net_device *dev, len_rthdr = ieee80211_get_radiotap_len(skb->data); hdr = (struct ieee80211_hdr *)(skb->data + len_rthdr); - if (skb->len < len_rthdr + 2 || - skb->len < len_rthdr + ieee80211_hdrlen(hdr->frame_control)) + if (skb_headlen(skb) < len_rthdr + 2 || + skb_headlen(skb) < len_rthdr + ieee80211_hdrlen(hdr->frame_control)) return 0; /* doesn't matter, frame will be dropped */ return ieee80211_select_queue_80211(sdata, skb, hdr); @@ -1023,6 +1023,13 @@ static int ieee80211_netdev_fill_forward_path(struct net_device_path_ctx *ctx, struct sta_info *sta; int ret = -ENOENT; + if (ctx->ieee80211) { + path->type = DEV_PATH_IEEE80211; + path->dev = ctx->dev; + ctx->dev = NULL; + return 0; + } + sdata = IEEE80211_DEV_TO_SUB_IF(ctx->dev); local = sdata->local; diff --git a/net/mac80211/mesh_pathtbl.c b/net/mac80211/mesh_pathtbl.c index 03171cf00855..07a21dfa8674 100644 --- a/net/mac80211/mesh_pathtbl.c +++ b/net/mac80211/mesh_pathtbl.c @@ -577,6 +577,12 @@ void mesh_fast_tx_cache(struct ieee80211_sub_if_data *sdata, goto unlock_sta; spin_lock(&cache->walk_lock); + if ((READ_ONCE(mpath->flags) & MESH_PATH_DELETED) || + (mppath && (READ_ONCE(mppath->flags) & MESH_PATH_DELETED))) { + kfree(entry); + goto unlock_cache; + } + prev = rhashtable_lookup_get_insert_fast(&cache->rht, &entry->rhash, fast_tx_rht_params); @@ -694,8 +700,10 @@ struct mesh_path *mesh_path_add(struct ieee80211_sub_if_data *sdata, return ERR_PTR(-ENOSPC); new_mpath = mesh_path_new(sdata, dst, GFP_ATOMIC); - if (!new_mpath) + if (!new_mpath) { + atomic_dec(&sdata->u.mesh.mpaths); return ERR_PTR(-ENOMEM); + } tbl = &sdata->u.mesh.mesh_paths; spin_lock_bh(&tbl->walk_lock); @@ -708,6 +716,7 @@ struct mesh_path *mesh_path_add(struct ieee80211_sub_if_data *sdata, if (mpath) { kfree(new_mpath); + atomic_dec(&sdata->u.mesh.mpaths); if (IS_ERR(mpath)) return mpath; @@ -733,10 +742,15 @@ int mpp_path_add(struct ieee80211_sub_if_data *sdata, if (is_multicast_ether_addr(dst)) return -EOPNOTSUPP; + if (!atomic_add_unless(&sdata->u.mesh.mpaths, 1, MESH_MAX_MPATHS)) + return -ENOSPC; + new_mpath = mesh_path_new(sdata, dst, GFP_ATOMIC); - if (!new_mpath) + if (!new_mpath) { + atomic_dec(&sdata->u.mesh.mpaths); return -ENOMEM; + } memcpy(new_mpath->mpp, mpp, ETH_ALEN); tbl = &sdata->u.mesh.mpp_paths; @@ -749,10 +763,12 @@ int mpp_path_add(struct ieee80211_sub_if_data *sdata, hlist_add_head_rcu(&new_mpath->walk_list, &tbl->walk_head); spin_unlock_bh(&tbl->walk_lock); - if (ret) + if (ret) { kfree(new_mpath); - else + atomic_dec(&sdata->u.mesh.mpaths); + } else { mesh_fast_tx_flush_addr(sdata, dst); + } sdata->u.mesh.mpp_paths_generation++; return ret; @@ -798,7 +814,8 @@ static void mesh_path_free_rcu(struct mesh_table *tbl, struct ieee80211_sub_if_data *sdata = mpath->sdata; spin_lock_bh(&mpath->state_lock); - mpath->flags |= MESH_PATH_RESOLVING | MESH_PATH_DELETED; + WRITE_ONCE(mpath->flags, + mpath->flags | MESH_PATH_RESOLVING | MESH_PATH_DELETED); mesh_gate_del(tbl, mpath); spin_unlock_bh(&mpath->state_lock); timer_shutdown_sync(&mpath->timer); @@ -812,6 +829,9 @@ static void __mesh_path_del(struct mesh_table *tbl, struct mesh_path *mpath) { hlist_del_rcu(&mpath->walk_list); rhashtable_remove_fast(&tbl->rhead, &mpath->rhash, mesh_rht_params); + spin_lock_bh(&mpath->state_lock); + WRITE_ONCE(mpath->flags, mpath->flags | MESH_PATH_DELETED); + spin_unlock_bh(&mpath->state_lock); if (tbl == &mpath->sdata->u.mesh.mpp_paths) mesh_fast_tx_flush_addr(mpath->sdata, mpath->dst); else diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index f51167f0fc46..3c31c16860ab 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -5877,6 +5877,7 @@ static bool ieee80211_assoc_config_link(struct ieee80211_link_data *link, bool is_6ghz = cbss->channel->band == NL80211_BAND_6GHZ; bool is_s1g = cbss->channel->band == NL80211_BAND_S1GHZ; const struct cfg80211_bss_ies *bss_ies = NULL; + struct ieee802_11_elems *bss_elems = NULL; struct ieee80211_supported_band *sband; struct ieee802_11_elems *elems; u16 capab_info; @@ -6007,7 +6008,6 @@ static bool ieee80211_assoc_config_link(struct ieee80211_link_data *link, (is_5ghz && link->u.mgd.conn.mode >= IEEE80211_CONN_MODE_VHT && (!elems->vht_cap_elem || !elems->vht_operation)))) { const struct cfg80211_bss_ies *ies; - struct ieee802_11_elems *bss_elems; rcu_read_lock(); ies = rcu_dereference(cbss->ies); @@ -6069,7 +6069,6 @@ static bool ieee80211_assoc_config_link(struct ieee80211_link_data *link, "AP bug: VHT operation missing from AssocResp\n"); } } - kfree(bss_elems); } /* @@ -6342,6 +6341,7 @@ static bool ieee80211_assoc_config_link(struct ieee80211_link_data *link, ret = true; out: kfree(elems); + kfree(bss_elems); kfree(bss_ies); return ret; } diff --git a/net/mac80211/rc80211_minstrel_ht.c b/net/mac80211/rc80211_minstrel_ht.c index b73ef3adfcc5..45e7a7ea7690 100644 --- a/net/mac80211/rc80211_minstrel_ht.c +++ b/net/mac80211/rc80211_minstrel_ht.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -1193,6 +1194,54 @@ minstrel_ht_update_stats(struct minstrel_priv *mp, struct minstrel_ht_sta *mi) mi->sample_time = jiffies; } +/* + * Check whether an HT/VHT rate from a TX status entry maps to an entry + * in the minstrel_ht rate tables. Values that cannot be represented + * there must not be used for indexing mi->groups[] and the MCS groups. + */ +static bool +minstrel_ht_txstat_rate_valid(struct ieee80211_tx_rate *rate) +{ + unsigned int bw; + + if (!(rate->flags & (IEEE80211_TX_RC_MCS | IEEE80211_TX_RC_VHT_MCS))) + return true; + + if (rate->flags & IEEE80211_TX_RC_MCS) { + /* minstrel_ht supports up to MINSTREL_MAX_STREAMS streams */ + return rate->idx < MINSTREL_MAX_STREAMS * 8; + } + + /* minstrel_ht has no VHT groups for 160 MHz and wider */ + bw = !!(rate->flags & IEEE80211_TX_RC_40_MHZ_WIDTH) + + 2 * !!(rate->flags & IEEE80211_TX_RC_80_MHZ_WIDTH); + if ((rate->flags & IEEE80211_TX_RC_160_MHZ_WIDTH) || bw > BW_80) + return false; + + return ieee80211_rate_get_vht_nss(rate) <= MINSTREL_MAX_STREAMS && + ieee80211_rate_get_vht_mcs(rate) < MCS_GROUP_RATES; +} + +static bool +minstrel_ht_ri_txstat_rate_valid(struct rate_info *rate) +{ + if (!(rate->flags & (RATE_INFO_FLAGS_MCS | RATE_INFO_FLAGS_VHT_MCS))) + return true; + + if (rate->flags & RATE_INFO_FLAGS_MCS) { + /* minstrel_ht supports up to MINSTREL_MAX_STREAMS streams */ + return rate->mcs < MINSTREL_MAX_STREAMS * 8; + } + + /* minstrel_ht has VHT groups only for 20/40/80 MHz */ + if (rate->bw != RATE_INFO_BW_20 && rate->bw != RATE_INFO_BW_40 && + rate->bw != RATE_INFO_BW_80) + return false; + + return rate->nss <= MINSTREL_MAX_STREAMS && + rate->mcs < MCS_GROUP_RATES; +} + static bool minstrel_ht_txstat_valid(struct minstrel_priv *mp, struct minstrel_ht_sta *mi, struct ieee80211_tx_rate *rate) @@ -1205,9 +1254,8 @@ minstrel_ht_txstat_valid(struct minstrel_priv *mp, struct minstrel_ht_sta *mi, if (!rate->count) return false; - if (rate->flags & IEEE80211_TX_RC_MCS || - rate->flags & IEEE80211_TX_RC_VHT_MCS) - return true; + if (rate->flags & (IEEE80211_TX_RC_MCS | IEEE80211_TX_RC_VHT_MCS)) + return minstrel_ht_txstat_rate_valid(rate); for (i = 0; i < ARRAY_SIZE(mp->cck_rates); i++) if (rate->idx == mp->cck_rates[i]) @@ -1235,9 +1283,9 @@ minstrel_ht_ri_txstat_valid(struct minstrel_priv *mp, if (!rate_status->try_count) return false; - if (rate_status->rate_idx.flags & RATE_INFO_FLAGS_MCS || - rate_status->rate_idx.flags & RATE_INFO_FLAGS_VHT_MCS) - return true; + if (rate_status->rate_idx.flags & + (RATE_INFO_FLAGS_MCS | RATE_INFO_FLAGS_VHT_MCS)) + return minstrel_ht_ri_txstat_rate_valid(&rate_status->rate_idx); for (i = 0; i < ARRAY_SIZE(mp->cck_rates); i++) { if (rate_status->rate_idx.legacy == @@ -1947,14 +1995,39 @@ minstrel_ht_alloc(struct ieee80211_hw *hw) } #ifdef CONFIG_MAC80211_DEBUGFS +static int minstrel_ht_fixed_rate_idx_get(void *data, u64 *val) +{ + *val = *(u32 *)data; + return 0; +} + +static int minstrel_ht_fixed_rate_idx_set(void *data, u64 val) +{ + u32 idx = val; + + /* U32_MAX is the default and keeps fixed rate processing disabled */ + if (val != U32_MAX && + (val > U16_MAX || + MI_RATE_GROUP(idx) >= ARRAY_SIZE(minstrel_mcs_groups) || + MI_RATE_IDX(idx) >= MCS_GROUP_RATES)) + return -EINVAL; + + *(u32 *)data = idx; + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(minstrel_ht_fixed_rate_idx_fops, + minstrel_ht_fixed_rate_idx_get, + minstrel_ht_fixed_rate_idx_set, "%llu\n"); + static void minstrel_ht_add_debugfs(struct ieee80211_hw *hw, void *priv, struct dentry *debugfsdir) { struct minstrel_priv *mp = priv; mp->fixed_rate_idx = (u32) -1; - debugfs_create_u32("fixed_rate_idx", S_IRUGO | S_IWUGO, debugfsdir, - &mp->fixed_rate_idx); + debugfs_create_file("fixed_rate_idx", S_IRUGO | S_IWUGO, debugfsdir, + &mp->fixed_rate_idx, &minstrel_ht_fixed_rate_idx_fops); } #endif diff --git a/net/mac80211/rx.c b/net/mac80211/rx.c index 5e26be8e27d8..7ead509304eb 100644 --- a/net/mac80211/rx.c +++ b/net/mac80211/rx.c @@ -2499,6 +2499,12 @@ ieee80211_rx_h_defragment(struct ieee80211_rx_data *rx) } skb_pull(rx->skb, ieee80211_hdrlen(fc)); + if (unlikely((u32)entry->extra_len + rx->skb->len > U16_MAX)) { + I802_DEBUG_INC(rx->local->rx_handlers_drop_defrag); + __skb_queue_purge(&entry->skb_list); + return RX_DROP_U_DEFRAG_OVERFLOW; + } + __skb_queue_tail(&entry->skb_list, rx->skb); entry->last_frag = frag; entry->extra_len += rx->skb->len; diff --git a/net/mac80211/spectmgmt.c b/net/mac80211/spectmgmt.c index 880f4625775d..f5e4970869b6 100644 --- a/net/mac80211/spectmgmt.c +++ b/net/mac80211/spectmgmt.c @@ -111,8 +111,8 @@ validate_chandef_by_ht_vht_oper(struct ieee80211_sub_if_data *sdata, switch (chan_width) { case NL80211_CHAN_WIDTH_320: - WARN_ON(1); - break; + chandef->chan = NULL; + return; case NL80211_CHAN_WIDTH_160: vht_oper.chan_width = IEEE80211_VHT_CHANWIDTH_80MHZ; vht_oper.center_freq_seg1_idx = vht_oper.center_freq_seg0_idx; diff --git a/net/mac80211/sta_info.c b/net/mac80211/sta_info.c index fdf00cbf49d8..e81cd155197b 100644 --- a/net/mac80211/sta_info.c +++ b/net/mac80211/sta_info.c @@ -137,6 +137,8 @@ static void __cleanup_single_sta(struct sta_info *sta) struct ieee80211_local *local = sdata->local; struct ps_data *ps; + cancel_work_sync(&sta->drv_deliver_wk); + if (test_sta_flag(sta, WLAN_STA_PS_STA) || test_sta_flag(sta, WLAN_STA_PS_DRIVER) || test_sta_flag(sta, WLAN_STA_PS_DELIVER)) { @@ -166,8 +168,6 @@ static void __cleanup_single_sta(struct sta_info *sta) if (ieee80211_vif_is_mesh(&sdata->vif)) mesh_sta_cleanup(sta); - cancel_work_sync(&sta->drv_deliver_wk); - /* * Destroy aggregation state here. It would be nice to wait for the * driver to finish aggregation stop and then clean up, but for now @@ -1582,6 +1582,8 @@ static void __sta_info_destroy_part2(struct sta_info *sta, bool recalc) sta->dead = true; + cancel_work_sync(&sta->drv_deliver_wk); + local->num_sta--; local->sta_generation++; diff --git a/net/mac80211/status.c b/net/mac80211/status.c index 3d811f652603..414979e2a1d9 100644 --- a/net/mac80211/status.c +++ b/net/mac80211/status.c @@ -1164,6 +1164,73 @@ void ieee80211_tx_status_skb(struct ieee80211_hw *hw, struct sk_buff *skb) } EXPORT_SYMBOL(ieee80211_tx_status_skb); +/* + * Check whether the HT/VHT rate information in a TX status entry is + * valid for its encoding. This is not specific to any rate control + * algorithm; all status consumers rely on the values being sane. + */ +static bool ieee80211_tx_status_rate_info_valid(const struct rate_info *rate) +{ + if (rate->flags & RATE_INFO_FLAGS_MCS) + return rate->mcs < 32; + + if (rate->flags & RATE_INFO_FLAGS_VHT_MCS) + return rate->nss >= 1 && rate->nss <= 8 && rate->mcs <= 11; + + return true; +} + +static bool +ieee80211_tx_status_tx_rate_valid(const struct ieee80211_tx_rate *rate) +{ + if (rate->flags & IEEE80211_TX_RC_MCS) + return rate->idx < 32; + + if (rate->flags & IEEE80211_TX_RC_VHT_MCS) + return ieee80211_rate_get_vht_mcs(rate) <= 11; + + return true; +} + +/* + * Drop TX status rate entries that don't describe a valid rate. The + * status information is used by rate control and by other mac80211 + * code, and a malformed entry must not be able to corrupt state beyond + * the driver that reported it. + */ +static void +ieee80211_tx_status_drop_invalid_rates(struct ieee80211_tx_status *status) +{ + int i; + + for (i = 0; i < status->n_rates; i++) { + struct ieee80211_rate_status *rs = &status->rates[i]; + + if (ieee80211_tx_status_rate_info_valid(&rs->rate_idx)) + continue; + + rs->try_count = 0; + memset(&rs->rate_idx, 0, sizeof(rs->rate_idx)); + } + + if (!status->info) + return; + + for (i = 0; i < IEEE80211_TX_MAX_RATES; i++) { + struct ieee80211_tx_rate *rate; + + rate = &status->info->status.rates[i]; + if (rate->idx < 0) + break; + + if (ieee80211_tx_status_tx_rate_valid(rate)) + continue; + + rate->idx = -1; + rate->count = 0; + } +} + void ieee80211_tx_status_ext(struct ieee80211_hw *hw, struct ieee80211_tx_status *status) { @@ -1176,6 +1243,8 @@ void ieee80211_tx_status_ext(struct ieee80211_hw *hw, bool acked, noack_success, ack_signal_valid; u16 tx_time_est; + ieee80211_tx_status_drop_invalid_rates(status); + if (pubsta) { sta = container_of(pubsta, struct sta_info, sta); @@ -1300,6 +1369,7 @@ void ieee80211_tx_rate_update(struct ieee80211_hw *hw, .sta = pubsta, }; + ieee80211_tx_status_drop_invalid_rates(&status); rate_control_tx_status(local, &status); if (ieee80211_hw_check(&local->hw, HAS_RATE_CONTROL)) diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c index 814399989b5e..d51f810dd2c5 100644 --- a/net/mac80211/tx.c +++ b/net/mac80211/tx.c @@ -2091,7 +2091,7 @@ static bool ieee80211_validate_radiotap_len(struct sk_buff *skb) (struct ieee80211_radiotap_header *)skb->data; /* check for not even having the fixed radiotap header part */ - if (unlikely(skb->len < sizeof(struct ieee80211_radiotap_header))) + if (unlikely(skb_headlen(skb) < sizeof(struct ieee80211_radiotap_header))) return false; /* too short to be possibly valid */ /* is it a header version we can trust to find length from? */ @@ -2099,7 +2099,7 @@ static bool ieee80211_validate_radiotap_len(struct sk_buff *skb) return false; /* only version 0 is supported */ /* does the skb contain enough to deliver on the alleged length? */ - if (unlikely(skb->len < ieee80211_get_radiotap_len(skb->data))) + if (unlikely(skb_headlen(skb) < ieee80211_get_radiotap_len(skb->data))) return false; /* skb too short for claimed rt header extent */ return true; @@ -2388,13 +2388,13 @@ netdev_tx_t ieee80211_monitor_start_xmit(struct sk_buff *skb, skb_set_network_header(skb, len_rthdr); skb_set_transport_header(skb, len_rthdr); - if (skb->len < len_rthdr + 2) + if (skb_headlen(skb) < len_rthdr + 2) goto fail; hdr = (struct ieee80211_hdr *)(skb->data + len_rthdr); hdrlen = ieee80211_hdrlen(hdr->frame_control); - if (skb->len < len_rthdr + hdrlen) + if (skb_headlen(skb) < len_rthdr + hdrlen) goto fail; /* @@ -2402,7 +2402,7 @@ netdev_tx_t ieee80211_monitor_start_xmit(struct sk_buff *skb, * carrying a rfc1042 header */ if (ieee80211_is_data(hdr->frame_control) && - skb->len >= len_rthdr + hdrlen + sizeof(rfc1042_header) + 2) { + skb_headlen(skb) >= len_rthdr + hdrlen + sizeof(rfc1042_header) + 2) { u8 *payload = (u8 *)hdr + hdrlen; if (ether_addr_equal(payload, rfc1042_header)) @@ -2532,7 +2532,7 @@ static inline bool ieee80211_is_tdls_setup(struct sk_buff *skb) int ieee80211_lookup_ra_sta(struct ieee80211_sub_if_data *sdata, struct sk_buff *skb, - struct sta_info **sta_out) + struct sta_info **sta_out, bool bss) { struct sta_info *sta; @@ -2553,7 +2553,10 @@ int ieee80211_lookup_ra_sta(struct ieee80211_sub_if_data *sdata, *sta_out = ERR_PTR(-ENOENT); return 0; } - sta = sta_info_get_bss(sdata, skb->data); + if (bss) + sta = sta_info_get_bss(sdata, skb->data); + else + sta = sta_info_get(sdata, skb->data); break; #ifdef CONFIG_MAC80211_MESH case NL80211_IFTYPE_MESH_POINT: @@ -4419,7 +4422,8 @@ void __ieee80211_subif_start_xmit(struct sk_buff *skb, ieee80211_mesh_xmit_fast(sdata, skb, ctrl_flags)) goto out; - if (ieee80211_lookup_ra_sta(sdata, skb, &sta)) + if (ieee80211_lookup_ra_sta(sdata, skb, &sta, + skb->protocol == sdata->control_port_protocol)) goto out_free; if (IS_ERR(sta)) @@ -4731,6 +4735,7 @@ static void ieee80211_8023_xmit(struct ieee80211_sub_if_data *sdata, { struct ieee80211_tx_info *info; struct ieee80211_local *local = sdata->local; + struct ieee80211_chanctx_conf *chanctx_conf; struct tid_ampdu_tx *tid_tx = NULL; struct sk_buff *seg, *next; unsigned int skbs = 0, len = 0; @@ -4780,6 +4785,16 @@ static void ieee80211_8023_xmit(struct ieee80211_sub_if_data *sdata, sdata = container_of(sdata->bss, struct ieee80211_sub_if_data, u.ap); + /* MLD transmissions must not rely on the band */ + if (!ieee80211_vif_is_mld(&sdata->vif)) { + chanctx_conf = rcu_dereference(sdata->vif.bss_conf.chanctx_conf); + if (unlikely(!chanctx_conf)) { + kfree_skb_list(skb); + return; + } + info->band = chanctx_conf->def.chan->band; + } + info->flags |= IEEE80211_TX_CTL_HW_80211_ENCAP; info->control.vif = &sdata->vif; @@ -4848,7 +4863,10 @@ static void __ieee80211_subif_start_xmit_8023(struct sk_buff *skb, rcu_read_lock(); - if (ieee80211_lookup_ra_sta(sdata, skb, &sta)) { + if (unlikely(sdata->control_port_protocol == ehdr->h_proto)) + goto skip_offload; + + if (ieee80211_lookup_ra_sta(sdata, skb, &sta, false)) { kfree_skb(skb); goto out; } @@ -4869,8 +4887,7 @@ static void __ieee80211_subif_start_xmit_8023(struct sk_buff *skb, link = &sdata->deflink; key = rcu_dereference(link->default_multicast_key); } else if (unlikely(IS_ERR_OR_NULL(sta) || !sta->uploaded || - !test_sta_flag(sta, WLAN_STA_AUTHORIZED) || - sdata->control_port_protocol == ehdr->h_proto)) { + !test_sta_flag(sta, WLAN_STA_AUTHORIZED))) { goto skip_offload; } else { key = rcu_dereference(sta->ptk[sta->ptk_idx]); @@ -4931,7 +4948,7 @@ ieee80211_build_data_template(struct ieee80211_sub_if_data *sdata, rcu_read_lock(); - if (ieee80211_lookup_ra_sta(sdata, skb, &sta)) { + if (ieee80211_lookup_ra_sta(sdata, skb, &sta, false)) { kfree_skb(skb); skb = ERR_PTR(-EINVAL); goto out; @@ -5002,7 +5019,7 @@ static bool ieee80211_tx_pending_skb(struct ieee80211_local *local, } result = ieee80211_tx(sdata, NULL, skb, true); } else if (info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP) { - if (ieee80211_lookup_ra_sta(sdata, skb, &sta)) { + if (ieee80211_lookup_ra_sta(sdata, skb, &sta, true)) { dev_kfree_skb(skb); return true; } @@ -6639,7 +6656,7 @@ int ieee80211_tx_control_port(struct wiphy *wiphy, struct net_device *dev, * AF_PACKET */ rcu_read_lock(); - err = ieee80211_lookup_ra_sta(sdata, skb, &sta); + err = ieee80211_lookup_ra_sta(sdata, skb, &sta, true); if (err) { dev_kfree_skb(skb); rcu_read_unlock(); diff --git a/net/mctp/device.c b/net/mctp/device.c index 822120e860c8..9f76ff092559 100644 --- a/net/mctp/device.c +++ b/net/mctp/device.c @@ -344,13 +344,13 @@ static struct mctp_dev *mctp_add_dev(struct net_device *dev) mdev->net = mctp_default_net(dev_net(dev)); - /* associate to net_device */ refcount_set(&mdev->refs, 1); - rcu_assign_pointer(dev->mctp_ptr, mdev); - dev_hold(dev); mdev->dev = dev; + /* Associate to net_device, publish the mdev. */ + rcu_assign_pointer(dev->mctp_ptr, mdev); + return mdev; } diff --git a/net/mctp/route.c b/net/mctp/route.c index b19c63a5691a..e7c95eeacb48 100644 --- a/net/mctp/route.c +++ b/net/mctp/route.c @@ -825,7 +825,7 @@ static struct mctp_sk_key *mctp_lookup_prealloc_tag(struct mctp_sock *msk, spin_lock_irqsave(&mns->keys_lock, flags); - hlist_for_each_entry(tmp, &mns->keys, hlist) { + hlist_for_each_entry(tmp, &msk->keys, sklist) { if (tmp->net != netid) continue; diff --git a/net/netfilter/ipset/ip_set_bitmap_gen.h b/net/netfilter/ipset/ip_set_bitmap_gen.h index d6a7e6604542..ae376fa3e7a3 100644 --- a/net/netfilter/ipset/ip_set_bitmap_gen.h +++ b/net/netfilter/ipset/ip_set_bitmap_gen.h @@ -159,7 +159,7 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext, if (SET_WITH_COUNTER(set)) ip_set_init_counter(ext_counter(x, set), ext); - if (SET_WITH_COMMENT(set)) + if (SET_WITH_COMMENT(set) && !ext->target) ip_set_init_comment(set, ext_comment(x, set), ext); if (SET_WITH_SKBINFO(set)) ip_set_init_skbinfo(ext_skbinfo(x, set), ext); diff --git a/net/netfilter/ipvs/ip_vs_conn.c b/net/netfilter/ipvs/ip_vs_conn.c index 6fa3e1dc534c..cb009208826f 100644 --- a/net/netfilter/ipvs/ip_vs_conn.c +++ b/net/netfilter/ipvs/ip_vs_conn.c @@ -1102,6 +1102,9 @@ ip_vs_bind_dest(struct ip_vs_conn *cp, struct ip_vs_dest *dest) if (cp->protocol != IPPROTO_UDP) conn_flags &= ~IP_VS_CONN_F_ONE_PACKET; flags = cp->flags; + /* Only visible templates can control multiple connections */ + if (flags & IP_VS_CONN_F_TEMPLATE) + conn_flags &= ~IP_VS_CONN_F_ONE_PACKET; /* Bind with the destination and its corresponding transmitter */ if (flags & IP_VS_CONN_F_SYNC) { /* Synced conns are hashed, so they can not get this flag */ diff --git a/net/netfilter/ipvs/ip_vs_core.c b/net/netfilter/ipvs/ip_vs_core.c index ba0957798bad..fd503f0efb57 100644 --- a/net/netfilter/ipvs/ip_vs_core.c +++ b/net/netfilter/ipvs/ip_vs_core.c @@ -1960,6 +1960,12 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, /* Ensure the IP header is present in headroom */ if (!pskb_may_pull(skb, hlen_orig)) goto ignore_tunnel; + skb_set_transport_header(skb, hlen_orig); + /* Before now we may used ihl from skb frag, revalidate it after + * copying it into skb head to prevent out-of-bounds access + */ + if (ip_hdr(skb)->ihl * 4 != hlen_orig) + goto ignore_tunnel; IP_VS_DBG(12, "Sending ICMP for %pI4->%pI4: t=%u, c=%u, i=%u\n", &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr, type, code, ntohl(info)); diff --git a/net/netfilter/ipvs/ip_vs_lblc.c b/net/netfilter/ipvs/ip_vs_lblc.c index 693bcc82ccb7..55af77a56929 100644 --- a/net/netfilter/ipvs/ip_vs_lblc.c +++ b/net/netfilter/ipvs/ip_vs_lblc.c @@ -203,7 +203,11 @@ ip_vs_lblc_new(struct ip_vs_lblc_table *tbl, const union nf_inet_addr *daddr, if (en->dest == dest) return en; ip_vs_lblc_del(en); + atomic_dec(&tbl->entries); } + if (atomic_read(&tbl->entries) >= tbl->max_size * 3 / 2) + return NULL; + en = kmalloc_obj(*en, GFP_ATOMIC); if (!en) return NULL; diff --git a/net/netfilter/ipvs/ip_vs_lblcr.c b/net/netfilter/ipvs/ip_vs_lblcr.c index f53f05ceea36..858393b1d2d1 100644 --- a/net/netfilter/ipvs/ip_vs_lblcr.c +++ b/net/netfilter/ipvs/ip_vs_lblcr.c @@ -363,6 +363,9 @@ ip_vs_lblcr_new(struct ip_vs_lblcr_table *tbl, const union nf_inet_addr *daddr, en = ip_vs_lblcr_get(af, tbl, daddr); if (!en) { + if (atomic_read(&tbl->entries) >= tbl->max_size * 3 / 2) + return NULL; + en = kmalloc_obj(*en, GFP_ATOMIC); if (!en) return NULL; diff --git a/net/netfilter/ipvs/ip_vs_sync.c b/net/netfilter/ipvs/ip_vs_sync.c index 5383aeafb0ae..69dc28153ec1 100644 --- a/net/netfilter/ipvs/ip_vs_sync.c +++ b/net/netfilter/ipvs/ip_vs_sync.c @@ -949,6 +949,21 @@ static void ip_vs_proc_conn(struct netns_ipvs *ipvs, struct ip_vs_conn_param *pa ip_vs_conn_put(cp); } +/* Check for incompatible flags */ +static bool ip_vs_sync_validate_flags(u32 flags) +{ + /* We do not expect NO_CPORT, especially to allow lookups + * to hit templates + */ + if (flags & IP_VS_CONN_F_NO_CPORT) { + if (flags & IP_VS_CONN_F_TEMPLATE) + return false; + } + if (flags & IP_VS_CONN_F_ONE_PACKET) + return false; + return true; +} + /* * Process received multicast message for Version 0 */ @@ -972,8 +987,7 @@ static void ip_vs_process_message_v0(struct netns_ipvs *ipvs, const char *buffer return; } s = (struct ip_vs_sync_conn_v0 *) p; - flags = ntohs(s->flags) | IP_VS_CONN_F_SYNC; - flags &= ~IP_VS_CONN_F_HASHED; + flags = ntohs(s->flags); if (flags & IP_VS_CONN_F_SEQ_MASK) { opt = (struct ip_vs_sync_conn_options *)&s[1]; p += FULL_CONN_SIZE; @@ -986,6 +1000,13 @@ static void ip_vs_process_message_v0(struct netns_ipvs *ipvs, const char *buffer p += SIMPLE_CONN_SIZE; } + if (!ip_vs_sync_validate_flags(flags)) { + IP_VS_DBG(2, "BACKUP v0, Invalid flags 0x%X\n", flags); + continue; + } + flags &= IP_VS_CONN_F_BACKUP_MASK; + flags |= IP_VS_CONN_F_SYNC; + state = ntohs(s->state); if (!(flags & IP_VS_CONN_F_TEMPLATE)) { pp = ip_vs_proto_get(s->protocol); @@ -1141,7 +1162,13 @@ static inline int ip_vs_proc_sync_conn(struct netns_ipvs *ipvs, __u8 *p, __u8 *m } /* Get flags and Mask off unsupported */ - flags = ntohl(s->v4.flags) & IP_VS_CONN_F_BACKUP_MASK; + flags = ntohl(s->v4.flags); + if (!ip_vs_sync_validate_flags(flags)) { + IP_VS_DBG(3, "BACKUP, Invalid flags 0x%X\n", flags); + retc = 25; + goto out; + } + flags &= IP_VS_CONN_F_BACKUP_MASK; flags |= IP_VS_CONN_F_SYNC; state = ntohs(s->v4.state); diff --git a/net/netfilter/nf_conntrack_netlink.c b/net/netfilter/nf_conntrack_netlink.c index 579ada063b1b..4e5d7c701436 100644 --- a/net/netfilter/nf_conntrack_netlink.c +++ b/net/netfilter/nf_conntrack_netlink.c @@ -3392,7 +3392,8 @@ static bool expect_iter_name(struct nf_conntrack_expect *exp, void *data) struct nf_conntrack_helper *helper; const char *name = data; - helper = rcu_dereference(exp->helper); + helper = rcu_dereference_protected(exp->helper, + lockdep_is_held(&nf_conntrack_expect_lock)); if (!helper) return false; diff --git a/net/netfilter/nf_flow_table_core.c b/net/netfilter/nf_flow_table_core.c index 934c6151f558..36bbc7be2f74 100644 --- a/net/netfilter/nf_flow_table_core.c +++ b/net/netfilter/nf_flow_table_core.c @@ -575,7 +575,12 @@ static void nf_flow_table_extend_ct_timeout(struct nf_conn *ct) static void nf_flow_offload_gc_step(struct nf_flowtable *flow_table, struct flow_offload *flow, void *data) { - bool teardown = test_bit(NF_FLOW_TEARDOWN, &flow->flags); + bool teardown; + + if (test_bit(NF_FLOW_PENDING, &flow->flags)) + return; + + teardown = test_bit(NF_FLOW_TEARDOWN, &flow->flags); if (nf_flow_has_expired(flow) || nf_ct_is_dying(flow->ct) || diff --git a/net/netfilter/nf_flow_table_offload.c b/net/netfilter/nf_flow_table_offload.c index 801a3dd9ceea..4365859220e6 100644 --- a/net/netfilter/nf_flow_table_offload.c +++ b/net/netfilter/nf_flow_table_offload.c @@ -1057,8 +1057,9 @@ static void flow_offload_work_handler(struct work_struct *work) default: WARN_ON_ONCE(1); } + smp_mb__before_atomic(); + clear_bit(NF_FLOW_PENDING, &offload->flow->flags); - clear_bit(NF_FLOW_HW_PENDING, &offload->flow->flags); kfree(offload); } @@ -1084,12 +1085,12 @@ nf_flow_offload_work_alloc(struct nf_flowtable *flowtable, { struct flow_offload_work *offload; - if (test_and_set_bit(NF_FLOW_HW_PENDING, &flow->flags)) + if (test_and_set_bit(NF_FLOW_PENDING, &flow->flags)) return NULL; offload = kmalloc_obj(struct flow_offload_work, GFP_ATOMIC); if (!offload) { - clear_bit(NF_FLOW_HW_PENDING, &flow->flags); + clear_bit(NF_FLOW_PENDING, &flow->flags); return NULL; } diff --git a/net/netfilter/nf_flow_table_path.c b/net/netfilter/nf_flow_table_path.c index 1e55644f2edb..d90013685bf1 100644 --- a/net/netfilter/nf_flow_table_path.c +++ b/net/netfilter/nf_flow_table_path.c @@ -53,6 +53,7 @@ static int nft_dev_fill_forward_path(const struct dst_entry *dst_cache, struct net_device_path_ctx ctx = { .dev = dev, .ether_type = ether_type, + .ieee80211 = true, }; struct neighbour *n; u8 nud_state; @@ -114,6 +115,7 @@ static int nft_dev_path_info(struct net_device_path_stack *stack, path = &stack->path[i]; switch (path->type) { case DEV_PATH_ETHERNET: + case DEV_PATH_IEEE80211: case DEV_PATH_DSA: case DEV_PATH_VLAN: case DEV_PATH_PPPOE: @@ -123,6 +125,7 @@ static int nft_dev_path_info(struct net_device_path_stack *stack, memcpy(info->h_source, path->dev->dev_addr, ETH_ALEN); if (path->type == DEV_PATH_ETHERNET || + path->type == DEV_PATH_IEEE80211 || path->type == DEV_PATH_DSA) break; diff --git a/net/netfilter/nf_nat_bpf.c b/net/netfilter/nf_nat_bpf.c index f9dd85ccea01..7572b58c448b 100644 --- a/net/netfilter/nf_nat_bpf.c +++ b/net/netfilter/nf_nat_bpf.c @@ -39,6 +39,9 @@ __bpf_kfunc int bpf_ct_set_nat_info(struct nf_conn___init *nfct, if (proto != NFPROTO_IPV4 && proto != NFPROTO_IPV6) return -EINVAL; + if (manip != NF_NAT_MANIP_SRC && manip != NF_NAT_MANIP_DST) + return -EINVAL; + memset(&range, 0, sizeof(struct nf_nat_range2)); range.flags = NF_NAT_RANGE_MAP_IPS; range.min_addr = *addr; diff --git a/net/netfilter/nf_nat_core.c b/net/netfilter/nf_nat_core.c index a4858c2b2d65..cc8e1e81006d 100644 --- a/net/netfilter/nf_nat_core.c +++ b/net/netfilter/nf_nat_core.c @@ -767,8 +767,9 @@ nf_nat_setup_info(struct nf_conn *ct, if (nf_ct_is_confirmed(ct)) return NF_ACCEPT; - WARN_ON(maniptype != NF_NAT_MANIP_SRC && - maniptype != NF_NAT_MANIP_DST); + if (WARN_ON(maniptype != NF_NAT_MANIP_SRC && + maniptype != NF_NAT_MANIP_DST)) + return NF_DROP; if (WARN_ON(nf_nat_initialized(ct, maniptype))) return NF_DROP; diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c index c0b754a2d45b..b59628e6240c 100644 --- a/net/netfilter/nf_tables_api.c +++ b/net/netfilter/nf_tables_api.c @@ -6995,11 +6995,14 @@ static int nft_setelem_catchall_insert(const struct net *net, { struct nft_set_elem_catchall *catchall; u8 genmask = nft_genmask_next(net); + u64 tstamp = nft_net_tstamp(net); struct nft_set_ext *ext; list_for_each_entry(catchall, &set->catchall_list, list) { ext = nft_set_elem_ext(set, catchall->elem); - if (nft_set_elem_active(ext, genmask)) { + if (nft_set_elem_active(ext, genmask) && + !__nft_set_elem_expired(ext, tstamp) && + !nft_set_elem_is_dead(ext)) { *priv = catchall->elem; return -EEXIST; } @@ -7092,11 +7095,14 @@ static int nft_setelem_catchall_deactivate(const struct net *net, struct nft_set_elem *elem) { struct nft_set_elem_catchall *catchall; + u64 tstamp = nft_net_tstamp(net); struct nft_set_ext *ext; list_for_each_entry(catchall, &set->catchall_list, list) { ext = nft_set_elem_ext(set, catchall->elem); - if (!nft_is_active_next(net, ext)) + if (!nft_is_active_next(net, ext) || + __nft_set_elem_expired(ext, tstamp) || + nft_set_elem_is_dead(ext)) continue; kfree(elem->priv); diff --git a/net/netfilter/nfnetlink_queue.c b/net/netfilter/nfnetlink_queue.c index c727668b0c5b..a3bc00280051 100644 --- a/net/netfilter/nfnetlink_queue.c +++ b/net/netfilter/nfnetlink_queue.c @@ -1593,6 +1593,7 @@ nfqnl_rcv_nl_event(struct notifier_block *this, if (event == NETLINK_URELEASE && n->protocol == NETLINK_NETFILTER) { int i; + nfnl_lock(NFNL_SUBSYS_QUEUE); /* destroy all instances for this portid */ spin_lock(&q->instances_lock); for (i = 0; i < INSTANCE_BUCKETS; i++) { @@ -1606,6 +1607,7 @@ nfqnl_rcv_nl_event(struct notifier_block *this, } } spin_unlock(&q->instances_lock); + nfnl_unlock(NFNL_SUBSYS_QUEUE); } return NOTIFY_DONE; } @@ -1925,9 +1927,9 @@ static int nfqnl_recv_config(struct sk_buff *skb, const struct nfnl_info *info, /* Lookup queue under RCU. After peer_portid check (or for new queue * in BIND case), the queue is owned by the socket sending this message. - * A socket cannot simultaneously send a message and close, so while - * processing this CONFIG message, nfqnl_rcv_nl_event() (triggered by - * socket close) cannot destroy this queue. Safe to use without RCU. + * nfqnl_rcv_nl_event() will block on the nfnl subsys mutex that is + * held by the caller, so the queue cannot be destroyed in parallel, + * even after we drop the RCU read lock. */ rcu_read_lock(); queue = instance_lookup(q, queue_num); diff --git a/net/netfilter/nft_flow_offload.c b/net/netfilter/nft_flow_offload.c index 32b4281038dd..d3c5651dd699 100644 --- a/net/netfilter/nft_flow_offload.c +++ b/net/netfilter/nft_flow_offload.c @@ -160,6 +160,7 @@ static int nft_flow_offload_init(const struct nft_ctx *ctx, struct nft_flow_offload *priv = nft_expr_priv(expr); u8 genmask = nft_genmask_next(ctx->net); struct nft_flowtable *flowtable; + int err; if (!tb[NFTA_FLOW_TABLE_NAME]) return -EINVAL; @@ -174,7 +175,11 @@ static int nft_flow_offload_init(const struct nft_ctx *ctx, priv->flowtable = flowtable; - return nf_ct_netns_get(ctx->net, ctx->family); + err = nf_ct_netns_get(ctx->net, ctx->family); + if (err < 0) + nft_use_dec(&flowtable->use); + + return err; } static void nft_flow_offload_deactivate(const struct nft_ctx *ctx, diff --git a/net/netfilter/nft_set_rbtree.c b/net/netfilter/nft_set_rbtree.c index 9894832281c4..12431b55752f 100644 --- a/net/netfilter/nft_set_rbtree.c +++ b/net/netfilter/nft_set_rbtree.c @@ -900,6 +900,8 @@ static void nft_rbtree_gc_scan(struct nft_set *set) next = rb_next(node); rbe = rb_entry(node, struct nft_rbtree_elem, node); + if (!nft_set_elem_active(&rbe->ext, NFT_GENMASK_ANY)) + continue; /* elements are reversed in the rbtree for historical reasons, * from highest to lowest value, that is why end element is diff --git a/net/netfilter/nft_synproxy.c b/net/netfilter/nft_synproxy.c index 9ed288c9d168..554a96a000f4 100644 --- a/net/netfilter/nft_synproxy.c +++ b/net/netfilter/nft_synproxy.c @@ -118,7 +118,8 @@ static void nft_synproxy_do_eval(const struct nft_synproxy *priv, return; } - if (nf_ip_checksum(skb, nft_hook(pkt), thoff, IPPROTO_TCP)) { + if (nf_checksum(skb, nft_hook(pkt), thoff, IPPROTO_TCP, + nft_pf(pkt))) { regs->verdict.code = NF_DROP; return; } diff --git a/net/netlink/genetlink.c b/net/netlink/genetlink.c index 41d37442f186..5cc1037d4917 100644 --- a/net/netlink/genetlink.c +++ b/net/netlink/genetlink.c @@ -1656,7 +1656,7 @@ static void *ctrl_dumppolicy_prep(struct sk_buff *skb, } static int ctrl_dumppolicy_put_op(struct sk_buff *skb, - struct netlink_callback *cb, + struct netlink_callback *cb, u32 cmd, struct genl_split_ops *doit, struct genl_split_ops *dumpit) { @@ -1677,7 +1677,7 @@ static int ctrl_dumppolicy_put_op(struct sk_buff *skb, if (!nest_pol) goto err; - nest_op = nla_nest_start(skb, doit->cmd); + nest_op = nla_nest_start(skb, cmd); if (!nest_op) goto err; @@ -1721,7 +1721,8 @@ static int ctrl_dumppolicy(struct sk_buff *skb, struct netlink_callback *cb) &doit, &dumpit))) return -ENOENT; - if (ctrl_dumppolicy_put_op(skb, cb, &doit, &dumpit)) + if (ctrl_dumppolicy_put_op(skb, cb, ctx->op, + &doit, &dumpit)) return skb->len; /* done with the per-op policy index list */ @@ -1730,6 +1731,7 @@ static int ctrl_dumppolicy(struct sk_buff *skb, struct netlink_callback *cb) while (ctx->dump_map) { if (ctrl_dumppolicy_put_op(skb, cb, + ctx->op_iter->cmd, &ctx->op_iter->doit, &ctx->op_iter->dumpit)) return skb->len; diff --git a/net/nfc/core.c b/net/nfc/core.c index a92a6566e6a0..f521669293f0 100644 --- a/net/nfc/core.c +++ b/net/nfc/core.c @@ -279,10 +279,10 @@ static struct nfc_target *nfc_find_target(struct nfc_dev *dev, u32 target_idx) int nfc_dep_link_up(struct nfc_dev *dev, int target_index, u8 comm_mode) { - int rc = 0; - u8 *gb; - size_t gb_len; struct nfc_target *target; + u8 gb[NFC_MAX_GT_LEN]; + size_t gb_len = 0; + int rc = 0; pr_debug("dev_name=%s comm %d\n", dev_name(&dev->dev), comm_mode); @@ -301,7 +301,7 @@ int nfc_dep_link_up(struct nfc_dev *dev, int target_index, u8 comm_mode) goto error; } - gb = nfc_llcp_general_bytes(dev, &gb_len); + nfc_get_local_general_bytes(dev, gb, sizeof(gb), &gb_len); if (gb_len > NFC_MAX_GT_LEN) { rc = -EINVAL; goto error; @@ -644,11 +644,10 @@ int nfc_set_remote_general_bytes(struct nfc_dev *dev, const u8 *gb, u8 gb_len) } EXPORT_SYMBOL(nfc_set_remote_general_bytes); -u8 *nfc_get_local_general_bytes(struct nfc_dev *dev, size_t *gb_len) +u8 *nfc_get_local_general_bytes(struct nfc_dev *dev, u8 *out_gb, + size_t gb_max_len, size_t *gb_len) { - pr_debug("dev_name=%s\n", dev_name(&dev->dev)); - - return nfc_llcp_general_bytes(dev, gb_len); + return nfc_llcp_general_bytes(dev, out_gb, gb_max_len, gb_len); } EXPORT_SYMBOL(nfc_get_local_general_bytes); diff --git a/net/nfc/digital_dep.c b/net/nfc/digital_dep.c index 3982fa084737..968547c306a5 100644 --- a/net/nfc/digital_dep.c +++ b/net/nfc/digital_dep.c @@ -1490,14 +1490,14 @@ static int digital_tg_send_atr_res(struct nfc_digital_dev *ddev, struct digital_atr_req *atr_req) { struct digital_atr_res *atr_res; + u8 gb[NFC_MAX_GT_LEN]; struct sk_buff *skb; - u8 *gb, payload_bits; + u8 payload_bits; size_t gb_len; int rc; - gb = nfc_get_local_general_bytes(ddev->nfc_dev, &gb_len); - if (!gb) - gb_len = 0; + nfc_get_local_general_bytes(ddev->nfc_dev, gb, sizeof(gb), + &gb_len); skb = digital_skb_alloc(ddev, sizeof(struct digital_atr_res) + gb_len); if (!skb) diff --git a/net/nfc/llcp.h b/net/nfc/llcp.h index d8345ed57c95..23ae7a0112d3 100644 --- a/net/nfc/llcp.h +++ b/net/nfc/llcp.h @@ -91,6 +91,7 @@ struct nfc_llcp_local { struct hlist_head pending_sdreqs; struct timer_list sdreq_timer; struct work_struct sdreq_timeout_work; + struct work_struct release_work; u8 sdreq_next_tid; /* sockets array */ diff --git a/net/nfc/llcp_commands.c b/net/nfc/llcp_commands.c index ca89fe967d6a..80a00938c869 100644 --- a/net/nfc/llcp_commands.c +++ b/net/nfc/llcp_commands.c @@ -135,7 +135,7 @@ struct nfc_llcp_sdp_tlv *nfc_llcp_build_sdreq_tlv(u8 tid, const char *uri, { struct nfc_llcp_sdp_tlv *sdreq; - pr_debug("uri: %s, len: %zu\n", uri, uri_len); + pr_debug("uri: %.*s, len: %zu\n", (int)uri_len, uri, uri_len); /* sdreq->tlv_len is u8, takes uri_len, + 3 for header, + 1 for NULL */ if (WARN_ON_ONCE(uri_len > U8_MAX - 4)) diff --git a/net/nfc/llcp_core.c b/net/nfc/llcp_core.c index cac1b5487064..74bf817007cf 100644 --- a/net/nfc/llcp_core.c +++ b/net/nfc/llcp_core.c @@ -20,6 +20,8 @@ static LIST_HEAD(llcp_devices); /* Protects llcp_devices list */ static DEFINE_SPINLOCK(llcp_devices_lock); +static struct workqueue_struct *llcp_wq; + static void nfc_llcp_rx_skb(struct nfc_llcp_local *local, struct sk_buff *skb); void nfc_llcp_sock_link(struct llcp_sock_list *l, struct sock *sk) @@ -63,21 +65,33 @@ static void nfc_llcp_socket_purge(struct nfc_llcp_sock *sock) } } +static struct sock *nfc_llcp_sock_list_pop(struct llcp_sock_list *l) +{ + struct sock *sk; + + write_lock(&l->lock); + sk = sk_head(&l->head); + if (sk) { + sock_hold(sk); + sk_del_node_init(sk); + } + write_unlock(&l->lock); + + return sk; +} + static void nfc_llcp_socket_release(struct nfc_llcp_local *local, bool device, int err) { struct sock *sk; - struct hlist_node *tmp; struct nfc_llcp_sock *llcp_sock; skb_queue_purge(&local->tx_queue); - write_lock(&local->sockets.lock); - - sk_for_each_safe(sk, tmp, &local->sockets.head) { + while ((sk = nfc_llcp_sock_list_pop(&local->sockets))) { llcp_sock = nfc_llcp_sock(sk); - bh_lock_sock(sk); + lock_sock(sk); nfc_llcp_socket_purge(llcp_sock); @@ -91,17 +105,27 @@ static void nfc_llcp_socket_release(struct nfc_llcp_local *local, bool device, list_for_each_entry_safe(lsk, n, &llcp_sock->accept_queue, accept_queue) { + bool put_creation = false; + accept_sk = &lsk->sk; - bh_lock_sock(accept_sk); + lock_sock_nested(accept_sk, + SINGLE_DEPTH_NESTING); - nfc_llcp_accept_unlink(accept_sk); + if (nfc_llcp_sock(accept_sk)->parent == sk) { + nfc_llcp_accept_unlink(accept_sk); + nfc_llcp_sock_unlink(&local->sockets, accept_sk); - if (err) - accept_sk->sk_err = err; - accept_sk->sk_state = LLCP_CLOSED; - accept_sk->sk_state_change(sk); + if (err) + accept_sk->sk_err = err; + accept_sk->sk_state = LLCP_CLOSED; + accept_sk->sk_state_change(accept_sk); + sock_orphan(accept_sk); + put_creation = true; + } - bh_unlock_sock(accept_sk); + release_sock(accept_sk); + if (put_creation) + sock_put(accept_sk); /* creation ref */ } } @@ -110,23 +134,18 @@ static void nfc_llcp_socket_release(struct nfc_llcp_local *local, bool device, sk->sk_state = LLCP_CLOSED; sk->sk_state_change(sk); - bh_unlock_sock(sk); - - sk_del_node_init(sk); + release_sock(sk); + sock_put(sk); } - write_unlock(&local->sockets.lock); - /* If we still have a device, we keep the RAW sockets alive */ if (device == true) return; - write_lock(&local->raw_sockets.lock); - - sk_for_each_safe(sk, tmp, &local->raw_sockets.head) { + while ((sk = nfc_llcp_sock_list_pop(&local->raw_sockets))) { llcp_sock = nfc_llcp_sock(sk); - bh_lock_sock(sk); + lock_sock(sk); nfc_llcp_socket_purge(llcp_sock); @@ -135,26 +154,20 @@ static void nfc_llcp_socket_release(struct nfc_llcp_local *local, bool device, sk->sk_state = LLCP_CLOSED; sk->sk_state_change(sk); - bh_unlock_sock(sk); - - sk_del_node_init(sk); + release_sock(sk); + sock_put(sk); } - - write_unlock(&local->raw_sockets.lock); } static struct nfc_llcp_local *nfc_llcp_local_get(struct nfc_llcp_local *local) { - /* Since using nfc_llcp_local may result in usage of nfc_dev, whenever - * we hold a reference to local, we also need to hold a reference to - * the device to avoid UAF. - */ - if (!nfc_get_device(local->dev->idx)) + if (!local) return NULL; - kref_get(&local->ref); + if (kref_get_unless_zero(&local->ref)) + return local; - return local; + return NULL; } static void local_cleanup(struct nfc_llcp_local *local) @@ -172,30 +185,34 @@ static void local_cleanup(struct nfc_llcp_local *local) nfc_llcp_free_sdp_tlv_list(&local->pending_sdreqs); } +static void local_release_work(struct work_struct *work) +{ + struct nfc_llcp_local *local; + struct nfc_dev *dev; + + local = container_of(work, struct nfc_llcp_local, release_work); + dev = local->dev; + + local_cleanup(local); + kfree(local); + nfc_put_device(dev); +} + static void local_release(struct kref *ref) { struct nfc_llcp_local *local; local = container_of(ref, struct nfc_llcp_local, ref); - local_cleanup(local); - kfree(local); + queue_work(llcp_wq, &local->release_work); } int nfc_llcp_local_put(struct nfc_llcp_local *local) { - struct nfc_dev *dev; - int ret; - - if (local == NULL) + if (!local) return 0; - dev = local->dev; - - ret = kref_put(&local->ref, local_release); - nfc_put_device(dev); - - return ret; + return kref_put(&local->ref, local_release); } static struct nfc_llcp_sock *nfc_llcp_sock_get(struct nfc_llcp_local *local, @@ -341,7 +358,7 @@ static int nfc_llcp_wks_sap(const char *service_name, size_t service_name_len) { int sap, num_wks; - pr_debug("%s\n", service_name); + pr_debug("%.*s\n", (int)service_name_len, service_name); if (service_name == NULL) return -EINVAL; @@ -352,7 +369,8 @@ static int nfc_llcp_wks_sap(const char *service_name, size_t service_name_len) if (wks[sap] == NULL) continue; - if (strncmp(wks[sap], service_name, service_name_len) == 0) + if (strlen(wks[sap]) == service_name_len && + !strncmp(wks[sap], service_name, service_name_len)) return sap; } @@ -635,23 +653,32 @@ static int nfc_llcp_build_gb(struct nfc_llcp_local *local) return ret; } -u8 *nfc_llcp_general_bytes(struct nfc_dev *dev, size_t *general_bytes_len) +u8 *nfc_llcp_general_bytes(struct nfc_dev *dev, u8 *out_gb, size_t gb_max_len, + size_t *general_bytes_len) { struct nfc_llcp_local *local; + if (!out_gb || !general_bytes_len) + return NULL; + local = nfc_llcp_find_local(dev); - if (local == NULL) { + if (!local) { *general_bytes_len = 0; return NULL; } nfc_llcp_build_gb(local); - *general_bytes_len = local->gb_len; + if (local->gb_len) { + *general_bytes_len = min_t(size_t, local->gb_len, gb_max_len); + memcpy(out_gb, local->gb, *general_bytes_len); + } else { + *general_bytes_len = 0; + } nfc_llcp_local_put(local); - return local->gb; + return out_gb; } int nfc_llcp_set_remote_gb(struct nfc_dev *dev, const u8 *gb, u8 gb_len) @@ -1074,6 +1101,9 @@ static void nfc_llcp_recv_hdlc(struct nfc_llcp_local *local, struct sock *sk; u8 dsap, ssap, ptype, ns, nr; + if (!pskb_may_pull(skb, LLCP_HEADER_SIZE + LLCP_SEQUENCE_SIZE)) + return; + ptype = nfc_llcp_ptype(skb); dsap = nfc_llcp_dsap(skb); ssap = nfc_llcp_ssap(skb); @@ -1251,6 +1281,7 @@ static void nfc_llcp_recv_dm(struct nfc_llcp_local *local, struct nfc_llcp_sock *llcp_sock; struct sock *sk; u8 dsap, ssap, reason; + bool connecting = false; dsap = nfc_llcp_dsap(skb); ssap = nfc_llcp_ssap(skb); @@ -1262,6 +1293,7 @@ static void nfc_llcp_recv_dm(struct nfc_llcp_local *local, case LLCP_DM_NOBOUND: case LLCP_DM_REJ: llcp_sock = nfc_llcp_connecting_sock_get(local, dsap); + connecting = true; break; default: @@ -1276,10 +1308,33 @@ static void nfc_llcp_recv_dm(struct nfc_llcp_local *local, sk = &llcp_sock->sk; + lock_sock(sk); + + /* Check if socket was destroyed whilst waiting for the lock */ + if (!sk_hashed(sk)) { + release_sock(sk); + nfc_llcp_sock_put(llcp_sock); + return; + } + + /* + * For DM(NOBOUND)/DM(REJ) the socket is still linked on the + * connecting_sockets list. Unlink it here, under the socket lock, + * before moving it to LLCP_CLOSED: llcp_sock_release() selects the + * list to unlink from by sk_state, so leaving a connecting socket + * in the CLOSED state would make it unlink from the wrong list and + * corrupt the connecting_sockets list / desync the socket refcount. + * This mirrors nfc_llcp_recv_cc(). + */ + if (connecting) + nfc_llcp_sock_unlink(&local->connecting_sockets, sk); + sk->sk_err = ENXIO; sk->sk_state = LLCP_CLOSED; sk->sk_state_change(sk); + release_sock(sk); + nfc_llcp_sock_put(llcp_sock); } @@ -1680,6 +1735,7 @@ int nfc_llcp_register_device(struct nfc_dev *ndev) INIT_WORK(&local->rx_work, nfc_llcp_rx_work); INIT_WORK(&local->timeout_work, nfc_llcp_timeout_work); + INIT_WORK(&local->release_work, local_release_work); rwlock_init(&local->sockets.lock); rwlock_init(&local->connecting_sockets.lock); @@ -1723,10 +1779,23 @@ void nfc_llcp_unregister_device(struct nfc_dev *dev) int __init nfc_llcp_init(void) { - return nfc_llcp_sock_init(); + int ret; + + llcp_wq = alloc_workqueue("nfc_llcp_wq", WQ_UNBOUND, 0); + if (!llcp_wq) + return -ENOMEM; + + ret = nfc_llcp_sock_init(); + if (ret) { + destroy_workqueue(llcp_wq); + return ret; + } + + return 0; } void nfc_llcp_exit(void) { nfc_llcp_sock_exit(); + destroy_workqueue(llcp_wq); } diff --git a/net/nfc/llcp_sock.c b/net/nfc/llcp_sock.c index 5558d8a4d48b..1e5ee4bcde68 100644 --- a/net/nfc/llcp_sock.c +++ b/net/nfc/llcp_sock.c @@ -392,11 +392,12 @@ void nfc_llcp_accept_unlink(struct sock *sk) pr_debug("state %d\n", sk->sk_state); - list_del_init(&llcp_sock->accept_queue); - sk_acceptq_removed(llcp_sock->parent); - llcp_sock->parent = NULL; - - sock_put(sk); + if (llcp_sock->parent) { + list_del_init(&llcp_sock->accept_queue); + sk_acceptq_removed(llcp_sock->parent); + llcp_sock->parent = NULL; + sock_put(sk); + } } void nfc_llcp_accept_enqueue(struct sock *parent, struct sock *sk) @@ -423,12 +424,20 @@ struct sock *nfc_llcp_accept_dequeue(struct sock *parent, list_for_each_entry_safe(lsk, n, &llcp_parent->accept_queue, accept_queue) { + struct nfc_llcp_local *local; + sk = &lsk->sk; - lock_sock(sk); + lock_sock_nested(sk, SINGLE_DEPTH_NESTING); if (sk->sk_state == LLCP_CLOSED) { - release_sock(sk); + local = nfc_llcp_sock(sk)->local; + nfc_llcp_accept_unlink(sk); + if (local) + nfc_llcp_sock_unlink(&local->sockets, sk); + sock_orphan(sk); + release_sock(sk); + sock_put(sk); continue; } @@ -464,7 +473,7 @@ static int llcp_sock_accept(struct socket *sock, struct socket *newsock, pr_debug("parent %p\n", sk); - lock_sock_nested(sk, SINGLE_DEPTH_NESTING); + lock_sock(sk); if (sk->sk_state != LLCP_LISTEN) { ret = -EBADFD; @@ -490,7 +499,12 @@ static int llcp_sock_accept(struct socket *sock, struct socket *newsock, release_sock(sk); timeo = schedule_timeout(timeo); - lock_sock_nested(sk, SINGLE_DEPTH_NESTING); + lock_sock(sk); + + if (sk->sk_state != LLCP_LISTEN) { + ret = -EBADFD; + break; + } } __set_current_state(TASK_RUNNING); remove_wait_queue(sk_sleep(sk), &wait); @@ -629,13 +643,24 @@ static int llcp_sock_release(struct socket *sock) list_for_each_entry_safe(lsk, n, &llcp_sock->accept_queue, accept_queue) { - accept_sk = &lsk->sk; - lock_sock(accept_sk); + bool put_creation = false; - nfc_llcp_send_disconnect(lsk); - nfc_llcp_accept_unlink(accept_sk); + accept_sk = &lsk->sk; + lock_sock_nested(accept_sk, SINGLE_DEPTH_NESTING); + + if (nfc_llcp_sock(accept_sk)->parent == sk) { + nfc_llcp_send_disconnect(lsk); + nfc_llcp_accept_unlink(accept_sk); + nfc_llcp_sock_unlink(&local->sockets, accept_sk); + + accept_sk->sk_state = LLCP_CLOSED; + sock_orphan(accept_sk); + put_creation = true; + } release_sock(accept_sk); + if (put_creation) + sock_put(accept_sk); /* creation ref */ } } @@ -734,12 +759,16 @@ static int llcp_sock_connect(struct socket *sock, struct sockaddr_unsized *_addr llcp_sock->service_name_len = min_t(unsigned int, addr->service_name_len, NFC_LLCP_MAX_SERVICE_NAME); - llcp_sock->service_name = kmemdup(addr->service_name, - llcp_sock->service_name_len, - GFP_KERNEL); - if (!llcp_sock->service_name) { - ret = -ENOMEM; - goto sock_llcp_release; + if (llcp_sock->service_name_len == 0) { + llcp_sock->service_name = NULL; + } else { + llcp_sock->service_name = kmemdup(addr->service_name, + llcp_sock->service_name_len, + GFP_KERNEL); + if (!llcp_sock->service_name) { + ret = -ENOMEM; + goto sock_llcp_release; + } } nfc_llcp_sock_link(&local->connecting_sockets, sk); diff --git a/net/nfc/nci/core.c b/net/nfc/nci/core.c index 5f46c4b5720f..73e3a96470ac 100644 --- a/net/nfc/nci/core.c +++ b/net/nfc/nci/core.c @@ -780,15 +780,15 @@ static int nci_set_local_general_bytes(struct nfc_dev *nfc_dev) { struct nci_dev *ndev = nfc_get_drvdata(nfc_dev); struct nci_set_config_param param; + u8 gb[NFC_MAX_GT_LEN]; int rc; - param.val = nfc_get_local_general_bytes(nfc_dev, ¶m.len); - if ((param.val == NULL) || (param.len == 0)) + nfc_get_local_general_bytes(nfc_dev, gb, sizeof(gb), + ¶m.len); + if (param.len == 0) return 0; - if (param.len > NFC_MAX_GT_LEN) - return -EINVAL; - + param.val = gb; param.id = NCI_PN_ATR_REQ_GEN_BYTES; rc = nci_request(ndev, nci_set_config_req, ¶m, diff --git a/net/nfc/netlink.c b/net/nfc/netlink.c index 0c58824cb150..224bdfa2dd0d 100644 --- a/net/nfc/netlink.c +++ b/net/nfc/netlink.c @@ -1181,7 +1181,7 @@ static int nfc_genl_llc_sdreq(struct sk_buff *skb, struct genl_info *info) if (rc != 0) { rc = -EINVAL; - goto put_local; + goto free_list; } if (!sdp_attrs[NFC_SDP_ATTR_URI]) @@ -1200,7 +1200,7 @@ static int nfc_genl_llc_sdreq(struct sk_buff *skb, struct genl_info *info) sdreq = nfc_llcp_build_sdreq_tlv(tid, uri, uri_len); if (sdreq == NULL) { rc = -ENOMEM; - goto put_local; + goto free_list; } tlvs_len += sdreq->tlv_len; @@ -1215,6 +1215,9 @@ static int nfc_genl_llc_sdreq(struct sk_buff *skb, struct genl_info *info) rc = nfc_llcp_send_snl_sdreq(local, &sdreq_list, tlvs_len); +free_list: + nfc_llcp_free_sdp_tlv_list(&sdreq_list); + put_local: nfc_llcp_local_put(local); diff --git a/net/nfc/nfc.h b/net/nfc/nfc.h index 0b1e6466f4fb..82c5dfdad10e 100644 --- a/net/nfc/nfc.h +++ b/net/nfc/nfc.h @@ -49,7 +49,8 @@ void nfc_llcp_mac_is_up(struct nfc_dev *dev, u32 target_idx, int nfc_llcp_register_device(struct nfc_dev *dev); void nfc_llcp_unregister_device(struct nfc_dev *dev); int nfc_llcp_set_remote_gb(struct nfc_dev *dev, const u8 *gb, u8 gb_len); -u8 *nfc_llcp_general_bytes(struct nfc_dev *dev, size_t *general_bytes_len); +u8 *nfc_llcp_general_bytes(struct nfc_dev *dev, u8 *out_gb, size_t gb_max_len, + size_t *general_bytes_len); int nfc_llcp_data_received(struct nfc_dev *dev, struct sk_buff *skb); struct nfc_llcp_local *nfc_llcp_find_local(struct nfc_dev *dev); int nfc_llcp_local_put(struct nfc_llcp_local *local); diff --git a/net/openvswitch/conntrack.c b/net/openvswitch/conntrack.c index 0f433688e17b..d3326edcabf7 100644 --- a/net/openvswitch/conntrack.c +++ b/net/openvswitch/conntrack.c @@ -734,6 +734,18 @@ static int __ovs_ct_lookup(struct net *net, struct sw_flow_key *key, enum ip_conntrack_info ctinfo; struct nf_conn *ct; + /* If the ct entry is not confirmed and shared with some other skb, + * e.g., a cloned one, we can't just modify it with the commit as we + * must not modify the extension set. Reset. + */ + if (cached && info->commit) { + ct = nf_ct_get(skb, &ctinfo); + if (ct && !nf_ct_is_confirmed(ct) && nf_ct_shared(ct)) { + nf_reset_ct(skb); + cached = false; + } + } + if (!cached) { struct nf_hook_state state = { .hook = NF_INET_PRE_ROUTING, @@ -766,8 +778,6 @@ static int __ovs_ct_lookup(struct net *net, struct sw_flow_key *key, ct = nf_ct_get(skb, &ctinfo); if (ct) { - bool add_helper = false; - /* Packets starting a new connection must be NATted before the * helper, so that the helper knows about the NAT. We enforce * this by delaying both NAT and helper calls for unconfirmed @@ -799,7 +809,6 @@ static int __ovs_ct_lookup(struct net *net, struct sw_flow_key *key, GFP_ATOMIC); if (err) return err; - add_helper = true; /* helper installed, add seqadj if NAT is required */ if (info->nat && !nfct_seqadj(ct)) { @@ -808,14 +817,14 @@ static int __ovs_ct_lookup(struct net *net, struct sw_flow_key *key, } } - /* Call the helper only if: - * - nf_conntrack_in() was executed above ("!cached") or a - * helper was just attached ("add_helper") for a confirmed - * connection, or - * - When committing an unconfirmed connection. + /* Call the helper only if nf_conntrack_in() was executed + * above ("!cached"). + * + * For unconfirmed connections it will be called later during + * commit as we need to have all the other extensions allocated + * before the call. */ - if ((nf_ct_is_confirmed(ct) ? !cached || add_helper : - info->commit)) { + if (nf_ct_is_confirmed(ct) && !cached) { int err = nf_ct_helper(skb, ct, ctinfo, info->family); err = verdict_to_errno(err); @@ -1019,6 +1028,14 @@ static int ovs_ct_commit(struct net *net, struct sw_flow_key *key, return err; nf_conn_act_ct_ext_add(skb, ct, ctinfo); + + /* Call the helpers now. We couldn't do this before as + * all the extensions must be allocated before the call. + */ + err = nf_ct_helper(skb, ct, ctinfo, info->family); + err = verdict_to_errno(err); + if (err) + return err; } else if (IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS) && labels_nonzero(&info->labels.mask)) { err = ovs_ct_set_labels(ct, key, &info->labels.value, diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 50cae32ae269..8865c1a42667 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -617,7 +617,7 @@ static int prb_calc_retire_blk_tmo(struct packet_sock *po, return DEFAULT_PRB_RETIRE_TOV; div = ecmd.base.speed / 1000; - mbits = (blk_size_in_bytes * 8) / (1024 * 1024); + mbits = (u64)blk_size_in_bytes * 8 / (1024 * 1024); if (div) mbits /= div; @@ -2530,26 +2530,6 @@ static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev, goto drop_n_restore; } -static void tpacket_destruct_skb(struct sk_buff *skb) -{ - struct packet_sock *po = pkt_sk(skb->sk); - - if (likely(po->tx_ring.pg_vec)) { - void *ph; - __u32 ts; - - ph = skb_zcopy_get_nouarg(skb); - - ts = __packet_set_timestamp(po, ph, skb); - __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts); - - packet_dec_pending(&po->tx_ring); - complete(&po->skb_completion); - } - - sock_wfree(skb); -} - static int __packet_snd_vnet_parse(struct virtio_net_hdr *vnet_hdr, size_t len) { if ((vnet_hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && @@ -2589,27 +2569,56 @@ static int packet_snd_vnet_parse(struct msghdr *msg, size_t *len, return 0; } +struct tpacket_uarg { + struct ubuf_info ubuf; + struct packet_sock *po; + void *ph; +}; + +static void tpacket_ubuf_complete(struct sk_buff *skb, struct ubuf_info *uarg, + bool success) +{ + struct tpacket_uarg *tu = container_of(uarg, struct tpacket_uarg, ubuf); + struct packet_sock *po = tu->po; + void *ph = tu->ph; + __u32 ts; + + DEBUG_NET_WARN_ON_ONCE(!skb); + + if (!refcount_dec_and_test(&uarg->refcnt)) + return; + + ts = __packet_set_timestamp(po, ph, skb); + __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts); + + packet_dec_pending(&po->tx_ring); + complete(&po->skb_completion); + + kfree(tu); + sk_free(&po->sk); +} + +static const struct ubuf_info_ops tpacket_ubuf_ops = { + .complete = tpacket_ubuf_complete, +}; + static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb, - void *frame, struct net_device *dev, void *data, int tp_len, + struct net_device *dev, void *data, int tp_len, __be16 proto, unsigned char *addr, int hlen, int copylen, int hard_header_len, const struct sockcm_cookie *sockc) { - union tpacket_uhdr ph; int to_write, offset, len, nr_frags, len_max; struct socket *sock = po->sk.sk_socket; struct page *page; int err; - ph.raw = frame; - skb->protocol = proto; skb->dev = dev; skb->priority = sockc->priority; skb->mark = sockc->mark; skb_set_delivery_type_by_clockid(skb, sockc->transmit_time, po->sk.sk_clockid); skb_setup_tx_timestamp(skb, sockc); - skb_zcopy_set_nouarg(skb, ph.raw); skb_reserve(skb, hlen); skb_reset_network_header(skb); @@ -2749,6 +2758,7 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) struct virtio_net_hdr vnet_hdr; bool has_vnet_hdr = false; struct sockcm_cookie sockc; + struct tpacket_uarg *uarg; __be16 proto; int err, reserve = 0; void *ph; @@ -2864,6 +2874,9 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) vnet_hdr.hdr_len); has_vnet_hdr = true; } + /* Prevent the byte count from wrapping into a negative errno */ + if (unlikely(tp_len > INT_MAX - len_sum)) + break; copylen = max_t(int, copylen, hard_header_len); skb = sock_alloc_send_skb(&po->sk, hlen + tlen + sizeof(struct sockaddr_ll) + @@ -2876,7 +2889,7 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) err = len_sum; goto out_status; } - tp_len = tpacket_fill_skb(po, skb, ph, dev, data, tp_len, proto, + tp_len = tpacket_fill_skb(po, skb, dev, data, tp_len, proto, addr, hlen, copylen, hard_header_len, &sockc); if (likely(tp_len >= 0) && @@ -2908,7 +2921,24 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) virtio_net_hdr_set_proto(skb, &vnet_hdr); } - skb->destructor = tpacket_destruct_skb; + uarg = kmalloc(sizeof(*uarg), GFP_KERNEL); + if (unlikely(!uarg)) { + if (likely(len_sum > 0)) + err = len_sum; + else + err = -ENOMEM; + goto out_status; + } + uarg->po = po; + uarg->ph = ph; + uarg->ubuf.ops = &tpacket_ubuf_ops; + uarg->ubuf.flags = SKBFL_ZEROCOPY_FRAG; + refcount_set(&uarg->ubuf.refcnt, 1); + + /* Hold a sk_wmem_alloc reference until completion */ + refcount_inc(&po->sk.sk_wmem_alloc); + skb_zcopy_init(skb, &uarg->ubuf); + __packet_set_status(po, ph, TP_STATUS_SENDING); packet_inc_pending(&po->tx_ring); @@ -4486,21 +4516,20 @@ static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order, bool tx_ring vec->len = block_nr; pg_vec = vec->pg_vec; + if (tx_ring) { + vec->deferred = kzalloc_obj(*vec->deferred, + GFP_KERNEL | __GFP_NOWARN); + if (!vec->deferred) + goto out_free_pgvec; + vec->deferred->vec = vec; + INIT_DELAYED_WORK(&vec->deferred->work, + packet_free_pg_vec_work); + } + for (i = 0; i < block_nr; i++) { pg_vec[i].buffer = alloc_one_pg_vec_page(order); if (unlikely(!pg_vec[i].buffer)) goto out_free_pgvec; - - if (tx_ring && !vec->deferred && - is_vmalloc_addr(pg_vec[i].buffer)) { - vec->deferred = kzalloc_obj(*vec->deferred, - GFP_KERNEL | __GFP_NOWARN); - if (!vec->deferred) - goto out_free_pgvec; - vec->deferred->vec = vec; - INIT_DELAYED_WORK(&vec->deferred->work, - packet_free_pg_vec_work); - } } out: diff --git a/net/rds/connection.c b/net/rds/connection.c index b6c4beb50eaf..c752a8623cfc 100644 --- a/net/rds/connection.c +++ b/net/rds/connection.c @@ -276,6 +276,12 @@ static struct rds_connection *__rds_conn_create(struct net *net, conn->c_trans = trans; + /* The transport may just have been swapped for loopback; size the + * set of paths - which is also what rds_conn_destroy() tears down + * again - by the transport the connection actually uses. + */ + npaths = (trans->t_mp_capable ? RDS_MPATH_WORKERS : 1); + init_waitqueue_head(&conn->c_hs_waitq); for (i = 0; i < npaths; i++) { __rds_conn_path_init(conn, &conn->c_path[i], diff --git a/net/rds/ib_frmr.c b/net/rds/ib_frmr.c index bd861191157b..8397aa4a17ac 100644 --- a/net/rds/ib_frmr.c +++ b/net/rds/ib_frmr.c @@ -204,19 +204,16 @@ static int rds_ib_map_frmr(struct rds_ib_device *rds_ibdev, */ rds_ib_teardown_mr(ibmr); - ibmr->sg = sg; - ibmr->sg_len = sg_len; - ibmr->sg_dma_len = 0; frmr->sg_byte_len = 0; - WARN_ON(ibmr->sg_dma_len); - ibmr->sg_dma_len = ib_dma_map_sg(dev, ibmr->sg, ibmr->sg_len, + ibmr->sg_dma_len = ib_dma_map_sg(dev, sg, sg_len, DMA_BIDIRECTIONAL); if (unlikely(!ibmr->sg_dma_len)) { pr_warn("RDS/IB: %s failed!\n", __func__); return -EBUSY; } - frmr->sg_byte_len = 0; + ibmr->sg = sg; + ibmr->sg_len = sg_len; frmr->dma_npages = 0; len = 0; @@ -264,6 +261,8 @@ static int rds_ib_map_frmr(struct rds_ib_device *rds_ibdev, ib_dma_unmap_sg(rds_ibdev->dev, ibmr->sg, ibmr->sg_len, DMA_BIDIRECTIONAL); ibmr->sg_dma_len = 0; + ibmr->sg = NULL; + ibmr->sg_len = 0; return ret; } diff --git a/net/sched/act_api.c b/net/sched/act_api.c index 3f653721c45f..e45a63be397c 100644 --- a/net/sched/act_api.c +++ b/net/sched/act_api.c @@ -758,7 +758,7 @@ static int tcf_idr_delete_index(struct tcf_idrinfo *idrinfo, u32 index) mutex_lock(&idrinfo->lock); p = idr_find(&idrinfo->action_idr, index); - if (!p) { + if (IS_ERR_OR_NULL(p)) { mutex_unlock(&idrinfo->lock); return -ENOENT; } diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c index 9080cb386c16..b48e6d2bb5a7 100644 --- a/net/sched/act_ct.c +++ b/net/sched/act_ct.c @@ -289,7 +289,7 @@ static bool tcf_ct_flow_is_outdated(const struct flow_offload *flow) { return test_bit(IPS_SEEN_REPLY_BIT, &flow->ct->status) && test_bit(IPS_HW_OFFLOAD_BIT, &flow->ct->status) && - !test_bit(NF_FLOW_HW_PENDING, &flow->flags) && + !test_bit(NF_FLOW_PENDING, &flow->flags) && !test_bit(NF_FLOW_HW_ESTABLISHED, &flow->flags); } @@ -432,11 +432,10 @@ static void tcf_ct_flow_table_add(struct tcf_ct_flow_table *ct_ft, if (test_and_set_bit(IPS_OFFLOAD_BIT, &ct->status)) return; + /* NULL if ct is dying (raced flush) or the atomic alloc failed. */ entry = flow_offload_alloc(ct); - if (!entry) { - WARN_ON_ONCE(1); + if (!entry) goto err_alloc; - } if (tcp) { ct->proto.tcp.seen[0].flags |= IP_CT_TCP_FLAG_BE_LIBERAL; @@ -980,14 +979,13 @@ TC_INDIRECT_SCOPE int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a, struct tcf_result *res) { struct net *net = dev_net(skb->dev); + bool cached, commit, clear, nat; enum ip_conntrack_info ctinfo; struct tcf_ct *c = to_ct(a); struct nf_conn *tmpl = NULL; struct nf_hook_state state; - bool cached, commit, clear; int nh_ofs, err, retval; struct tcf_ct_params *p; - bool add_helper = false; bool skb_is_ours = false; bool skip_add = false; bool defrag = false; @@ -999,6 +997,7 @@ TC_INDIRECT_SCOPE int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a, retval = p->action; commit = p->ct_action & TCA_CT_ACT_COMMIT; clear = p->ct_action & TCA_CT_ACT_CLEAR; + nat = p->ct_action & TCA_CT_ACT_NAT; tmpl = p->tmpl; tcf_lastuse_update(&c->tcf_tm); @@ -1047,6 +1046,19 @@ TC_INDIRECT_SCOPE int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a, * different zone. */ cached = tcf_ct_skb_nfct_cached(net, skb, p); + + /* If the ct entry is not confirmed and shared with some other skb, + * e.g., a cloned one, we can't just modify it with a commit or nat + * as we must not modify the extension set. Reset. + */ + if (cached && (commit || nat)) { + ct = nf_ct_get(skb, &ctinfo); + if (ct && !nf_ct_is_confirmed(ct) && nf_ct_shared(ct)) { + nf_reset_ct(skb); + cached = false; + } + } + if (!cached) { if (tcf_ct_flow_table_lookup(p, skb, family)) { skip_add = true; @@ -1083,26 +1095,32 @@ TC_INDIRECT_SCOPE int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a, err = __nf_ct_try_assign_helper(ct, p->tmpl, GFP_ATOMIC); if (err) goto drop; - add_helper = true; - if (p->ct_action & TCA_CT_ACT_NAT && !nfct_seqadj(ct)) { + + if (nat && !nfct_seqadj(ct)) { if (!nfct_seqadj_ext_add(ct)) goto drop; } } - if (nf_ct_is_confirmed(ct) ? ((!cached && !skip_add) || add_helper) : commit) { - err = nf_ct_helper(skb, ct, ctinfo, family); - if (err != NF_ACCEPT) - goto nf_error; - } - if (commit) { tcf_ct_act_set_mark(ct, p->mark, p->mark_mask); tcf_ct_act_set_labels(ct, p->labels, p->labels_mask); if (!nf_ct_is_confirmed(ct)) nf_conn_act_ct_ext_add(skb, ct, ctinfo); + } + /* Run helpers for the connection if nf_conntrack_in() was executed + * or if we're about to commit. This has to be done after all the + * extensions are already added. + */ + if (nf_ct_is_confirmed(ct) ? (!cached && !skip_add) : commit) { + err = nf_ct_helper(skb, ct, ctinfo, family); + if (err != NF_ACCEPT) + goto nf_error; + } + + if (commit) { /* This will take care of sending queued events * even if the connection is already confirmed. */ diff --git a/net/sched/act_gate.c b/net/sched/act_gate.c index 5d228a402204..6d6d45e03c07 100644 --- a/net/sched/act_gate.c +++ b/net/sched/act_gate.c @@ -681,7 +681,35 @@ static void tcf_gate_stats_update(struct tc_action *a, u64 bytes, u64 packets, static size_t tcf_gate_get_fill_size(const struct tc_action *act) { - return nla_total_size(sizeof(struct tc_gate)); + struct tcf_gate *gact = to_gate(act); + const struct tcf_gate_params *p; + struct tcfg_gate_entry *entry; + size_t size = nla_total_size(sizeof(struct tc_gate)) /* TCA_GATE_PARMS */ + + 3 * nla_total_size_64bit(sizeof(u64)) /* TCA_GATE_BASE_TIME + * TCA_GATE_CYCLE_TIME + * TCA_GATE_CYCLE_TIME_EXT + */ + + nla_total_size(sizeof(s32)) /* TCA_GATE_CLOCKID */ + + nla_total_size(sizeof(u32)) /* TCA_GATE_FLAGS */ + + nla_total_size(sizeof(s32)) /* TCA_GATE_PRIORITY */ + + nla_total_size(0); /* TCA_GATE_ENTRY_LIST */ + /* TCA_GATE_TM is budgeted by tcf_action_shared_attrs_size() */ + + rcu_read_lock(); + p = rcu_dereference(gact->param); + if (p) { + list_for_each_entry_rcu(entry, &p->entries, list) + /* TCA_GATE_ONE_ENTRY nest and its attributes */ + size += nla_total_size(0) + + nla_total_size(sizeof(u32)) /* TCA_GATE_ENTRY_INDEX */ + + nla_total_size(0) /* TCA_GATE_ENTRY_GATE */ + + nla_total_size(sizeof(u32)) /* TCA_GATE_ENTRY_INTERVAL */ + + nla_total_size(sizeof(s32)) /* TCA_GATE_ENTRY_MAX_OCTETS */ + + nla_total_size(sizeof(s32)); /* TCA_GATE_ENTRY_IPV */ + } + rcu_read_unlock(); + + return size; } static void tcf_gate_entry_destructor(void *priv) diff --git a/net/sched/act_ife.c b/net/sched/act_ife.c index 9cea71fc1db3..2afd68983ece 100644 --- a/net/sched/act_ife.c +++ b/net/sched/act_ife.c @@ -737,6 +737,7 @@ static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a, u8 *curr_data; u16 mtype; u16 dlen; + int ret; curr_data = ife_tlv_meta_decode(tlv_data, ifehdr_end, &mtype, &dlen, NULL); @@ -745,13 +746,19 @@ static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a, return TC_ACT_SHOT; } - if (find_decode_metaid(skb, p, mtype, dlen, curr_data)) { - /* abuse overlimits to count when we receive metadata - * but dont have an ops for it + ret = find_decode_metaid(skb, p, mtype, dlen, curr_data); + if (ret < 0) { + /* abuse overlimits to count metadata we cannot + * decode: no ops for it, or the decoder rejected it */ - pr_info_ratelimited("Unknown metaid %d dlen %d\n", - mtype, dlen); qstats_cpu_overlimit_inc(ife->common.cpu_qstats); + + if (ret == -ENOENT) + pr_info_ratelimited("Unknown metaid %d dlen %d\n", + mtype, dlen); + else + pr_info_ratelimited("Failed to decode metaid %d dlen %d err %d\n", + mtype, dlen, ret); } } diff --git a/net/sched/act_meta_mark.c b/net/sched/act_meta_mark.c index ea0573cb8b2d..e2f61b22bf0f 100644 --- a/net/sched/act_meta_mark.c +++ b/net/sched/act_meta_mark.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -28,9 +29,10 @@ static int skbmark_encode(struct sk_buff *skb, void *skbdata, static int skbmark_decode(struct sk_buff *skb, void *data, u16 len) { - u32 ifemark = *(u32 *)data; + if (len != sizeof(u32)) + return -EINVAL; - skb->mark = ntohl(ifemark); + skb->mark = get_unaligned_be32(data); return 0; } diff --git a/net/sched/act_meta_skbprio.c b/net/sched/act_meta_skbprio.c index 2df3133ce5ad..5cdb57931eab 100644 --- a/net/sched/act_meta_skbprio.c +++ b/net/sched/act_meta_skbprio.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -33,9 +34,10 @@ static int skbprio_encode(struct sk_buff *skb, void *skbdata, static int skbprio_decode(struct sk_buff *skb, void *data, u16 len) { - u32 ifeprio = *(u32 *)data; + if (len != sizeof(u32)) + return -EINVAL; - skb->priority = ntohl(ifeprio); + skb->priority = get_unaligned_be32(data); return 0; } diff --git a/net/sched/act_meta_skbtcindex.c b/net/sched/act_meta_skbtcindex.c index 44547caead46..8803710c0905 100644 --- a/net/sched/act_meta_skbtcindex.c +++ b/net/sched/act_meta_skbtcindex.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -28,9 +29,10 @@ static int skbtcindex_encode(struct sk_buff *skb, void *skbdata, static int skbtcindex_decode(struct sk_buff *skb, void *data, u16 len) { - u16 ifetc_index = *(u16 *)data; + if (len != sizeof(u16)) + return -EINVAL; - skb->tc_index = ntohs(ifetc_index); + skb->tc_index = get_unaligned_be16(data); return 0; } diff --git a/net/sched/act_skbedit.c b/net/sched/act_skbedit.c index bfec6b668410..26258b353cde 100644 --- a/net/sched/act_skbedit.c +++ b/net/sched/act_skbedit.c @@ -72,9 +72,8 @@ TC_INDIRECT_SCOPE int tcf_skbedit_act(struct sk_buff *skb, } if (params->flags & SKBEDIT_F_QUEUE_MAPPING && skb->dev->real_num_tx_queues > params->queue_mapping) { -#ifdef CONFIG_NET_EGRESS - netdev_xmit_skip_txqueue(true); -#endif + if (skb_at_tc_egress(skb)) + netdev_xmit_skip_txqueue(true); skb_set_queue_mapping(skb, tcf_skbedit_hash(params, skb)); } if (params->flags & SKBEDIT_F_MARK) { diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index c47d2ee13641..a9f54988561f 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -2463,7 +2463,20 @@ static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n, } errout: - if (err && tp_state == TP_CREATED) + if (err && !IS_ERR_OR_NULL(tp) && + (tp_state == TP_CREATED || tp->ops->delete_empty)) + /* + * The request is dropping its reference to tp. If it was + * the last user (the idr is empty now), reclaim the proto. + * A tp this request created is reclaimed unconditionally: + * it is the only owner, so marking it for deletion is + * safe. Otherwise only classifiers with a delete_empty + * callback are reclaimed -- the callback admits an empty + * proto only, so a live shared proto is never unlinked. + * Classifiers without one (deleting is set + * unconditionally) are rtnl-serialized, so the raced + * window this guard closes cannot arise for them. + */ tcf_chain_tp_delete_empty(chain, tp, rtnl_held, NULL); errout_tp: if (chain) { diff --git a/net/sched/cls_u32.c b/net/sched/cls_u32.c index a3e65c8cf29e..76ce2d124079 100644 --- a/net/sched/cls_u32.c +++ b/net/sched/cls_u32.c @@ -1003,8 +1003,16 @@ static int u32_change(struct net *net, struct sk_buff *in_skb, return -ENOMEM; } } else { - err = idr_alloc_u32(&tp_c->handle_idr, ht, &handle, - handle, GFP_KERNEL); + /* The IDR is keyed on the mapped id, and that is + * what the destroy paths remove. Ask for it here, + * so a manual handle colliding with the + * auto-allocated id space is rejected (-ENOSPC) + * instead of aliasing a future auto id. + */ + u32 id = handle2id(handle); + + err = idr_alloc_u32(&tp_c->handle_idr, ht, &id, id, + GFP_KERNEL); if (err) { kfree(ht); return err; diff --git a/net/sched/em_text.c b/net/sched/em_text.c index 343f1aebeec2..4132f8c3c5fc 100644 --- a/net/sched/em_text.c +++ b/net/sched/em_text.c @@ -113,7 +113,7 @@ static void em_text_destroy(struct tcf_ematch *m) static int em_text_dump(struct sk_buff *skb, struct tcf_ematch *m) { struct text_match *tm = EM_TEXT_PRIV(m); - struct tcf_em_text conf; + struct tcf_em_text conf = {}; strscpy(conf.algo, tm->config->ops->name); conf.from_offset = tm->from_offset; diff --git a/net/sched/sch_codel.c b/net/sched/sch_codel.c index 6aa5829d6961..a1ba2e355f2d 100644 --- a/net/sched/sch_codel.c +++ b/net/sched/sch_codel.c @@ -205,7 +205,7 @@ static int codel_init(struct Qdisc *sch, struct nlattr *opt, codel_params_init(&q->params); codel_vars_init(&q->vars); codel_stats_init(&q->stats); - q->params.mtu = clamp_t(u32, psched_mtu(qdisc_dev(sch)), 256, 1 << 20); + q->params.mtu = min_t(u32, psched_mtu(qdisc_dev(sch)), 1 << 20); if (opt) { int err = codel_change(sch, opt, extack); diff --git a/net/sched/sch_fq_codel.c b/net/sched/sch_fq_codel.c index 969b2510b0b8..e6c87a32950f 100644 --- a/net/sched/sch_fq_codel.c +++ b/net/sched/sch_fq_codel.c @@ -517,8 +517,8 @@ static int fq_codel_init(struct Qdisc *sch, struct nlattr *opt, q->flows_cnt = 1024; q->memory_limit = 32 << 20; /* 32 MBytes */ q->drop_batch_size = 64; - mtu = clamp_t(u32, psched_mtu(qdisc_dev(sch)), 256, FQ_CODEL_QUANTUM_MAX); - q->quantum = mtu; + mtu = min_t(u32, psched_mtu(qdisc_dev(sch)), FQ_CODEL_QUANTUM_MAX); + q->quantum = clamp_t(u32, mtu, 256, FQ_CODEL_QUANTUM_MAX); INIT_LIST_HEAD(&q->new_flows); INIT_LIST_HEAD(&q->old_flows); codel_params_init(&q->cparams); diff --git a/net/sched/sch_hfsc.c b/net/sched/sch_hfsc.c index e87f5021a199..284490fd6ca9 100644 --- a/net/sched/sch_hfsc.c +++ b/net/sched/sch_hfsc.c @@ -386,6 +386,15 @@ cftree_update(struct hfsc_class *cl) #define SM_MASK ((1ULL << SM_SHIFT) - 1) #define ISM_MASK ((1ULL << ISM_SHIFT) - 1) +/* + * Cap on the non-descending hops a classify walk may take before its + * filter chain is treated as misconfigured. A flowid binding that was + * legal at bind time can become lateral once hfsc_adjust_levels() + * raises a class level; a few such hops are legitimate, an unbounded + * run means the chain cycles. + */ +#define HFSC_CLASSIFY_MAX_DRIFT 8 + static inline u64 seg_x2y(u64 x, u64 sm) { @@ -1133,6 +1142,7 @@ hfsc_classify(struct sk_buff *skb, struct Qdisc *sch, int *qerr) struct hfsc_class *head, *cl; struct tcf_result res; struct tcf_proto *tcf; + unsigned int drift; int result; if (TC_H_MAJ(skb->priority ^ sch->handle) == 0 && @@ -1142,6 +1152,7 @@ hfsc_classify(struct sk_buff *skb, struct Qdisc *sch, int *qerr) *qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS; head = &q->root; + drift = HFSC_CLASSIFY_MAX_DRIFT; tcf = rcu_dereference_bh(q->root.filter_list); while (tcf && (result = tcf_classify_qdisc(skb, tcf, &res, false)) >= 0) { #ifdef CONFIG_NET_CLS_ACT @@ -1167,6 +1178,17 @@ hfsc_classify(struct sk_buff *skb, struct Qdisc *sch, int *qerr) if (cl->level == 0) return cl; /* hit leaf class */ + /* + * flowid binds skip the level check above (res.class is set + * at bind time and levels drift after), so a walk can follow + * lateral hops without descending; a bounded number of them + * is legal, more means the chain cycles. + */ + if (cl->level >= head->level && drift-- == 0) { + pr_warn_ratelimited("hfsc: classify hop budget exhausted, dropping packet\n"); + return NULL; + } + /* apply inner filter chain */ tcf = rcu_dereference_bh(cl->filter_list); head = cl; diff --git a/net/sched/sch_teql.c b/net/sched/sch_teql.c index 9e52afc2d980..409ce50cc0db 100644 --- a/net/sched/sch_teql.c +++ b/net/sched/sch_teql.c @@ -265,14 +265,11 @@ __teql_resolve(struct sk_buff *skb, struct sk_buff *skb_res, } if (neigh_event_send(n, skb_res) == 0) { - int err; char haddr[MAX_ADDR_LEN]; neigh_ha_snapshot(haddr, n, dev); - err = dev_hard_header(skb, dev, ntohs(skb_protocol(skb, false)), - haddr, NULL, skb->len); - - if (err < 0) + if (dev_hard_header(skb, dev, ntohs(skb_protocol(skb, false)), + haddr, NULL, skb->len) < 0) err = -EINVAL; } else { err = (skb_res == NULL) ? -EAGAIN : 1; diff --git a/net/sctp/associola.c b/net/sctp/associola.c index c0512c827d0f..4521be3bd85a 100644 --- a/net/sctp/associola.c +++ b/net/sctp/associola.c @@ -1289,18 +1289,19 @@ void sctp_assoc_update_retran_path(struct sctp_association *asoc) /* Manually skip the head element. */ if (&trans->transports == &asoc->peer.transport_addr_list) continue; - if (trans->state == SCTP_UNCONFIRMED) - continue; - trans_next = sctp_trans_elect_best(trans, trans_next); - /* Active is good enough for immediate return. */ - if (trans_next->state == SCTP_ACTIVE) - break; + if (trans->state != SCTP_UNCONFIRMED) { + trans_next = sctp_trans_elect_best(trans, trans_next); + /* Active is good enough for immediate return. */ + if (trans_next->state == SCTP_ACTIVE) + break; + } /* We've reached the end, time to update path. */ if (trans == asoc->peer.retran_path) break; } - asoc->peer.retran_path = trans_next; + if (trans_next) + asoc->peer.retran_path = trans_next; pr_debug("%s: association:%p updated new path to addr:%pISpc\n", __func__, asoc, &asoc->peer.retran_path->ipaddr.sa); diff --git a/net/sctp/input.c b/net/sctp/input.c index 864741fae418..9494cfa51106 100644 --- a/net/sctp/input.c +++ b/net/sctp/input.c @@ -436,9 +436,10 @@ void sctp_icmp_proto_unreachable(struct sock *sk, if (timer_pending(&t->proto_unreach_timer)) return; else { - if (!mod_timer(&t->proto_unreach_timer, - jiffies + (HZ/20))) - sctp_transport_hold(t); + sctp_transport_hold(t); + if (mod_timer(&t->proto_unreach_timer, + jiffies + (HZ / 20))) + sctp_transport_put(t); } } else { struct net *net = sock_net(sk); diff --git a/net/sctp/sm_sideeffect.c b/net/sctp/sm_sideeffect.c index 0d99b7e8c082..35f540fb15fc 100644 --- a/net/sctp/sm_sideeffect.c +++ b/net/sctp/sm_sideeffect.c @@ -244,8 +244,9 @@ void sctp_generate_t3_rtx_event(struct timer_list *t) pr_debug("%s: sock is busy\n", __func__); /* Try again later. */ - if (!mod_timer(&transport->T3_rtx_timer, jiffies + (HZ/20))) - sctp_transport_hold(transport); + sctp_transport_hold(transport); + if (mod_timer(&transport->T3_rtx_timer, jiffies + (HZ / 20))) + sctp_transport_put(transport); goto out_unlock; } @@ -280,8 +281,9 @@ static void sctp_generate_timeout_event(struct sctp_association *asoc, timeout_type); /* Try again later. */ - if (!mod_timer(&asoc->timers[timeout_type], jiffies + (HZ/20))) - sctp_association_hold(asoc); + sctp_association_hold(asoc); + if (mod_timer(&asoc->timers[timeout_type], jiffies + (HZ / 20))) + sctp_association_put(asoc); goto out_unlock; } @@ -378,8 +380,9 @@ void sctp_generate_heartbeat_event(struct timer_list *t) pr_debug("%s: sock is busy\n", __func__); /* Try again later. */ - if (!mod_timer(&transport->hb_timer, jiffies + (HZ/20))) - sctp_transport_hold(transport); + sctp_transport_hold(transport); + if (mod_timer(&transport->hb_timer, jiffies + (HZ / 20))) + sctp_transport_put(transport); goto out_unlock; } @@ -388,8 +391,9 @@ void sctp_generate_heartbeat_event(struct timer_list *t) timeout = sctp_transport_timeout(transport); if (elapsed < timeout) { elapsed = timeout - elapsed; - if (!mod_timer(&transport->hb_timer, jiffies + elapsed)) - sctp_transport_hold(transport); + sctp_transport_hold(transport); + if (mod_timer(&transport->hb_timer, jiffies + elapsed)) + sctp_transport_put(transport); goto out_unlock; } @@ -422,9 +426,10 @@ void sctp_generate_proto_unreach_event(struct timer_list *t) pr_debug("%s: sock is busy\n", __func__); /* Try again later. */ - if (!mod_timer(&transport->proto_unreach_timer, - jiffies + (HZ/20))) - sctp_transport_hold(transport); + sctp_transport_hold(transport); + if (mod_timer(&transport->proto_unreach_timer, + jiffies + (HZ / 20))) + sctp_transport_put(transport); goto out_unlock; } @@ -458,8 +463,9 @@ void sctp_generate_reconf_event(struct timer_list *t) pr_debug("%s: sock is busy\n", __func__); /* Try again later. */ - if (!mod_timer(&transport->reconf_timer, jiffies + (HZ / 20))) - sctp_transport_hold(transport); + sctp_transport_hold(transport); + if (mod_timer(&transport->reconf_timer, jiffies + (HZ / 20))) + sctp_transport_put(transport); goto out_unlock; } @@ -495,8 +501,9 @@ void sctp_generate_probe_event(struct timer_list *t) pr_debug("%s: sock is busy\n", __func__); /* Try again later. */ - if (!mod_timer(&transport->probe_timer, jiffies + (HZ / 20))) - sctp_transport_hold(transport); + sctp_transport_hold(transport); + if (mod_timer(&transport->probe_timer, jiffies + (HZ / 20))) + sctp_transport_put(transport); goto out_unlock; } diff --git a/net/sctp/sm_statefuns.c b/net/sctp/sm_statefuns.c index c701cff6ea67..8dc65498763c 100644 --- a/net/sctp/sm_statefuns.c +++ b/net/sctp/sm_statefuns.c @@ -2654,6 +2654,8 @@ static enum sctp_disposition sctp_sf_do_5_2_6_stale( sctp_add_cmd_sf(commands, SCTP_CMD_REPLY, SCTP_CHUNK(reply)); + sctp_add_cmd_sf(commands, SCTP_CMD_DISCARD_PACKET, SCTP_NULL()); + return SCTP_DISPOSITION_CONSUME; nomem: diff --git a/net/sctp/socket.c b/net/sctp/socket.c index c7b9e325ec1c..4652fd90d9a6 100644 --- a/net/sctp/socket.c +++ b/net/sctp/socket.c @@ -1844,7 +1844,7 @@ static int sctp_sendmsg_to_asoc(struct sctp_association *asoc, if (asoc->ep->intl_enable) { timeo = sock_sndtimeo(sk, 0); err = sctp_wait_for_connect(asoc, &timeo); - if (err) { + if (err || sk->sk_shutdown & RCV_SHUTDOWN) { err = -ESRCH; goto err; } diff --git a/net/smc/smc_core.c b/net/smc/smc_core.c index 04aedd957543..9974149659c2 100644 --- a/net/smc/smc_core.c +++ b/net/smc/smc_core.c @@ -1849,6 +1849,7 @@ void smcr_port_err(struct smc_ib_device *smcibdev, u8 ibport) struct smc_link_group *lgr, *n; int i; + spin_lock_bh(&smc_lgr_list.lock); list_for_each_entry_safe(lgr, n, &smc_lgr_list.list, list) { if (strncmp(smcibdev->pnetid[ibport - 1], lgr->pnet_id, SMC_MAX_PNETID_LEN)) @@ -1863,6 +1864,7 @@ void smcr_port_err(struct smc_ib_device *smcibdev, u8 ibport) smcr_link_down_cond_sched(lnk); } } + spin_unlock_bh(&smc_lgr_list.lock); } static void smc_link_down_work(struct work_struct *work) diff --git a/net/smc/smc_ib.c b/net/smc/smc_ib.c index 9bb495707445..daaa8a72da90 100644 --- a/net/smc/smc_ib.c +++ b/net/smc/smc_ib.c @@ -333,6 +333,7 @@ static bool smc_ib_check_link_gid(u8 gid[SMC_GID_SIZE], bool smcrv2, static void smc_ib_gid_check(struct smc_ib_device *smcibdev, u8 ibport) { struct smc_link_group *lgr; + bool stale_gid = false; int i; spin_lock_bh(&smc_lgr_list.lock); @@ -348,11 +349,16 @@ static void smc_ib_gid_check(struct smc_ib_device *smcibdev, u8 ibport) continue; if (!smc_ib_check_link_gid(lgr->lnk[i].gid, lgr->smc_version == SMC_V2, - smcibdev, ibport)) - smcr_port_err(smcibdev, ibport); + smcibdev, ibport)) { + stale_gid = true; + goto out; + } } } +out: spin_unlock_bh(&smc_lgr_list.lock); + if (stale_gid) + smcr_port_err(smcibdev, ibport); } static int smc_ib_remember_port_attr(struct smc_ib_device *smcibdev, u8 ibport) diff --git a/net/tipc/crypto.c b/net/tipc/crypto.c index 16f1ed1f6b1b..4409bdb70fe9 100644 --- a/net/tipc/crypto.c +++ b/net/tipc/crypto.c @@ -144,7 +144,7 @@ struct tipc_tfm { * @rcu: struct rcu_head * @key: the aead key * @gen: the key's generation - * @seqno: the key seqno (cluster scope) + * @seqno: the per-key TX nonce counter * @refcnt: the key reference counter */ struct tipc_aead { @@ -190,7 +190,6 @@ struct tipc_crypto_stats { * @rekeying_intv: rekeying interval (in minutes) * @stats: the crypto statistics * @name: the crypto name - * @sndnxt: the per-peer sndnxt (TX) * @timer1: general timer 1 (jiffies) * @timer2: general timer 2 (jiffies) * @working: the crypto is working or not @@ -219,7 +218,6 @@ struct tipc_crypto { struct tipc_crypto_stats __percpu *stats; char name[48]; - atomic64_t sndnxt ____cacheline_aligned; unsigned long timer1; unsigned long timer2; union { @@ -1051,14 +1049,11 @@ static int tipc_ehdr_build(struct net *net, struct tipc_aead *aead, WARN_ON(skb_headroom(skb) < ehsz); ehdr = (struct tipc_ehdr *)skb_push(skb, ehsz); - /* Obtain a seqno first: - * Use the key seqno (= cluster wise) if dest is unknown or we're in - * cluster key mode, otherwise it's better for a per-peer seqno! + /* + * Keep the nonce unique for the lifetime of the TX key, + * including key state changes and peer reconnection. */ - if (!__rx || aead->mode == CLUSTER_KEY) - seqno = atomic64_inc_return(&aead->seqno); - else - seqno = atomic64_inc_return(&__rx->sndnxt); + seqno = atomic64_inc_return(&aead->seqno); /* Revoke the key if seqno is wrapped around */ if (unlikely(!seqno)) @@ -1237,7 +1232,6 @@ void tipc_crypto_key_flush(struct tipc_crypto *c) tipc_crypto_key_set_state(c, 0, 0, 0); for (k = KEY_MIN; k <= KEY_MAX; k++) tipc_crypto_key_detach(c->aead[k], &c->lock); - atomic64_set(&c->sndnxt, 0); spin_unlock_bh(&c->lock); } @@ -1384,8 +1378,6 @@ static struct tipc_aead *tipc_crypto_key_pick_tx(struct tipc_crypto *tx, * It also considers if peer has no key, then we need to make own master key * (if any) taking over i.e. starting grace period and also trigger key * distributing process. - * - * The "per-peer" sndnxt is also reset when the peer key has switched. */ static void tipc_crypto_key_synch(struct tipc_crypto *rx, struct sk_buff *skb) { @@ -1436,7 +1428,6 @@ static void tipc_crypto_key_synch(struct tipc_crypto *rx, struct sk_buff *skb) if (cur) tipc_aead_users_dec(tx->aead[cur], 0); - atomic64_set(&rx->sndnxt, 0); /* Mark the point TX key users changed */ tx->timer1 = jiffies; @@ -1501,7 +1492,6 @@ int tipc_crypto_start(struct tipc_crypto **crypto, struct net *net, tipc_crypto_key_set_state(c, 0, 0, 0); atomic_set(&c->key_distr, 0); atomic_set(&c->peer_rx_active, 0); - atomic64_set(&c->sndnxt, 0); c->timer1 = jiffies; c->timer2 = jiffies; c->rekeying_intv = TIPC_REKEYING_INTV_DEF; diff --git a/net/tipc/group.c b/net/tipc/group.c index 14e6732624e2..74f6d3dac078 100644 --- a/net/tipc/group.c +++ b/net/tipc/group.c @@ -797,10 +797,10 @@ void tipc_group_proto_rcv(struct tipc_group *grp, bool *usr_wakeup, tipc_group_open(m, usr_wakeup); return; case GRP_ACK_MSG: - if (!m) + if (!m || !grp->bc_ackers) return; acked = msg_grp_bc_acked(hdr); - if (less_eq(acked, m->bc_acked)) + if (acked != grp->bc_snd_nxt || m->bc_acked == acked) return; m->bc_acked = acked; if (--grp->bc_ackers) diff --git a/net/tipc/monitor.c b/net/tipc/monitor.c index a94b9b36a700..1a438e312d66 100644 --- a/net/tipc/monitor.c +++ b/net/tipc/monitor.c @@ -632,9 +632,10 @@ static void mon_timeout(struct timer_list *t) { struct tipc_monitor *mon = timer_container_of(mon, t, timer); struct tipc_peer *self; - int best_member_cnt = dom_size(mon->peer_cnt) - 1; + int best_member_cnt; write_lock_bh(&mon->lock); + best_member_cnt = dom_size(mon->peer_cnt) - 1; self = mon->self; if (self && (best_member_cnt != self->applied)) { mon_update_local_domain(mon); diff --git a/net/vmw_vsock/af_vsock.c b/net/vmw_vsock/af_vsock.c index f840498b58af..9b71479a2b29 100644 --- a/net/vmw_vsock/af_vsock.c +++ b/net/vmw_vsock/af_vsock.c @@ -2889,6 +2889,9 @@ static int vsock_net_child_mode_string(const struct ctl_table *table, int write, net = container_of(table->data, struct net, vsock.child_ns_mode); + if (!*lenp) + return 0; + ret = __vsock_net_mode_string(table, write, buffer, lenp, ppos, vsock_net_child_mode(net), &new_mode); if (ret) diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index 9fafd7673d85..ddce61ec016a 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -4563,12 +4563,13 @@ static int nl80211_set_wiphy(struct sk_buff *skb, struct genl_info *info) rdev->wiphy.retry_long = retry_long; if (changed & WIPHY_PARAM_FRAG_THRESHOLD) rdev->wiphy.frag_threshold = frag_threshold; - if ((changed & WIPHY_PARAM_RTS_THRESHOLD) && - old_radio_rts_threshold) { + if (changed & WIPHY_PARAM_RTS_THRESHOLD) { rdev->wiphy.rts_threshold = rts_threshold; - for (i = 0 ; i < rdev->wiphy.n_radio; i++) - rdev->wiphy.radio_cfg[i].rts_threshold = - rdev->wiphy.rts_threshold; + if (old_radio_rts_threshold) { + for (i = 0; i < rdev->wiphy.n_radio; i++) + rdev->wiphy.radio_cfg[i].rts_threshold = + rdev->wiphy.rts_threshold; + } } if (changed & WIPHY_PARAM_COVERAGE_CLASS) rdev->wiphy.coverage_class = coverage_class; diff --git a/net/wireless/scan.c b/net/wireless/scan.c index caa9c6495f20..d35c826c4b1a 100644 --- a/net/wireless/scan.c +++ b/net/wireless/scan.c @@ -1992,20 +1992,22 @@ cfg80211_update_known_bss(struct cfg80211_registered_device *rdev, return false; } - old = rcu_access_pointer(known->pub.beacon_ies); + if (!new->pub.hidden_beacon_bss) { + old = rcu_access_pointer(known->pub.beacon_ies); - rcu_assign_pointer(known->pub.beacon_ies, new->pub.beacon_ies); + rcu_assign_pointer(known->pub.beacon_ies, new->pub.beacon_ies); - /* Override IEs if they were from a beacon before */ - if (old == rcu_access_pointer(known->pub.ies)) - rcu_assign_pointer(known->pub.ies, new->pub.beacon_ies); + /* Override IEs if they were from a beacon before */ + if (old == rcu_access_pointer(known->pub.ies)) + rcu_assign_pointer(known->pub.ies, new->pub.beacon_ies); - cfg80211_update_hidden_bsses(known, - rcu_access_pointer(new->pub.beacon_ies), - old); + cfg80211_update_hidden_bsses(known, + rcu_access_pointer(new->pub.beacon_ies), + old); - if (old) - kfree_rcu((struct cfg80211_bss_ies *)old, rcu_head); + if (old) + kfree_rcu((struct cfg80211_bss_ies *)old, rcu_head); + } } known->pub.beacon_interval = new->pub.beacon_interval; diff --git a/net/xdp/xskmap.c b/net/xdp/xskmap.c index 3bff346308d0..bf00d6463c19 100644 --- a/net/xdp/xskmap.c +++ b/net/xdp/xskmap.c @@ -124,7 +124,7 @@ static int xsk_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) struct bpf_insn *insn = insn_buf; *insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0); - *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 5); + *insn++ = BPF_JMP32_IMM(BPF_JGE, ret, map->max_entries, 5); *insn++ = BPF_ALU64_IMM(BPF_LSH, ret, ilog2(sizeof(struct xsk_sock *))); *insn++ = BPF_ALU64_IMM(BPF_ADD, mp, offsetof(struct xsk_map, xsk_map)); *insn++ = BPF_ALU64_REG(BPF_ADD, ret, mp); diff --git a/rust/kernel/net/netlink.rs b/rust/kernel/net/netlink.rs index 22ef3dde36fa..2f8733a694a9 100644 --- a/rust/kernel/net/netlink.rs +++ b/rust/kernel/net/netlink.rs @@ -11,6 +11,7 @@ use kernel::{ alloc::{self, AllocError}, error::to_result, + num::casts::u16_as_usize, prelude::*, transmute::AsBytes, types::Opaque, @@ -86,9 +87,17 @@ fn put(&mut self, attrtype: c_int, value: &T) -> Result where T: ?Sized + AsBytes, { + // `nla_len` is a 16-bit field that encodes the total attribute length + // (header + payload). Subtracting the header size from `u16::MAX` gives + // the largest payload that still fits within that field. + const MAX_PAYLOAD_LEN: usize = u16_as_usize(u16::MAX) - size_of::(); + let skb = self.skb.skb.as_ptr(); let len = size_of_val(value); let ptr = core::ptr::from_ref(value).cast::(); + if len > MAX_PAYLOAD_LEN { + return Err(EMSGSIZE); + } // SAFETY: `skb` is valid by `NetlinkSkBuff` type invariants, and the provided value is // readable and initialized for its `size_of` bytes. to_result(unsafe { bindings::nla_put(skb, attrtype, len as c_int, ptr) }) diff --git a/scripts/coccinelle/hid/ff_race.cocci b/scripts/coccinelle/hid/ff_race.cocci index 479f5d1e3184..e861de00cb10 100644 --- a/scripts/coccinelle/hid/ff_race.cocci +++ b/scripts/coccinelle/hid/ff_race.cocci @@ -7,18 +7,19 @@ virtual report -@r@ +@r exists@ identifier probe_fn; -expression hdev, flags; +identifier hdev; +expression flags; position p1, p2; @@ probe_fn(struct hid_device *hdev, ...) { - <... + ... when any hid_hw_start@p1(hdev, flags) ... - \(input_ff_create\|input_ff_create_memless\)@p2(...) - ...> + \(input_ff_create@p2\|input_ff_create_memless@p2\)(...) + ... when any } @script:python depends on report@ diff --git a/security/ipe/eval.c b/security/ipe/eval.c index 21439c5be336..4a7be96c84f0 100644 --- a/security/ipe/eval.c +++ b/security/ipe/eval.c @@ -134,10 +134,14 @@ static bool evaluate_boot_verified(const struct ipe_eval_ctx *const ctx) static bool evaluate_dmv_roothash(const struct ipe_eval_ctx *const ctx, struct ipe_prop *p) { - return !!ctx->ipe_bdev && - !!ctx->ipe_bdev->root_hash && - ipe_digest_eval(p->value, - ctx->ipe_bdev->root_hash); + const struct digest_info *root_hash; + + if (!ctx->ipe_bdev) + return false; + + root_hash = rcu_dereference(ctx->ipe_bdev->root_hash); + + return root_hash && ipe_digest_eval(p->value, root_hash); } #else static bool evaluate_dmv_roothash(const struct ipe_eval_ctx *const ctx, diff --git a/security/ipe/eval.h b/security/ipe/eval.h index fef65a36468c..b26d1147d070 100644 --- a/security/ipe/eval.h +++ b/security/ipe/eval.h @@ -27,7 +27,7 @@ struct ipe_bdev { #ifdef CONFIG_IPE_PROP_DM_VERITY_SIGNATURE bool dm_verity_signed; #endif /* CONFIG_IPE_PROP_DM_VERITY_SIGNATURE */ - struct digest_info *root_hash; + struct digest_info __rcu *root_hash; }; #endif /* CONFIG_IPE_PROP_DM_VERITY */ diff --git a/security/ipe/fs.c b/security/ipe/fs.c index 076c111c85c8..847a76afb93d 100644 --- a/security/ipe/fs.c +++ b/security/ipe/fs.c @@ -159,18 +159,16 @@ static ssize_t new_policy(struct file *f, const char __user *data, } rc = ipe_new_policyfs_node(p); - if (rc) - goto out; out: kfree(copy); if (rc < 0) { ipe_free_policy(p); ipe_audit_policy_load(ERR_PTR(rc)); - } else { - ipe_audit_policy_load(p); + return rc; } - return (rc < 0) ? rc : len; + + return len; } static const struct file_operations np_fops = { diff --git a/security/ipe/hooks.c b/security/ipe/hooks.c index 0ae54a880405..d878b52ceec8 100644 --- a/security/ipe/hooks.c +++ b/security/ipe/hooks.c @@ -9,6 +9,7 @@ #include #include #include +#include #include "ipe.h" #include "hooks.h" @@ -232,7 +233,20 @@ void ipe_bdev_free_security(struct block_device *bdev) { struct ipe_bdev *blob = ipe_bdev(bdev); - ipe_digest_free(blob->root_hash); + ipe_digest_free(rcu_access_pointer(blob->root_hash)); +} + +static void ipe_set_dmverity_roothash(struct ipe_bdev *blob, + struct digest_info *info) +{ + struct digest_info *old; + + /* Protected by device-mapper's md->suspend_lock */ + old = rcu_replace_pointer(blob->root_hash, info, true); + if (old) { + synchronize_rcu(); + ipe_digest_free(old); + } } #ifdef CONFIG_IPE_PROP_DM_VERITY_SIGNATURE @@ -280,8 +294,7 @@ int ipe_bdev_setintegrity(struct block_device *bdev, enum lsm_integrity_type typ return -EINVAL; if (!value) { - ipe_digest_free(blob->root_hash); - blob->root_hash = NULL; + ipe_set_dmverity_roothash(blob, NULL); return 0; } @@ -301,8 +314,7 @@ int ipe_bdev_setintegrity(struct block_device *bdev, enum lsm_integrity_type typ info->digest_len = digest->digest_len; - ipe_digest_free(blob->root_hash); - blob->root_hash = info; + ipe_set_dmverity_roothash(blob, info); return 0; err: diff --git a/security/ipe/policy_fs.c b/security/ipe/policy_fs.c index 9d92d8a14b13..a7aeb57483c6 100644 --- a/security/ipe/policy_fs.c +++ b/security/ipe/policy_fs.c @@ -481,6 +481,9 @@ int ipe_new_policyfs_node(struct ipe_policy *p) inode_lock(root); p->policyfs = policyfs; root->i_private = p; + /* Only audit signed policies from userspace */ + if (p->pkcs7) + ipe_audit_policy_load(p); inode_unlock(root); return 0; diff --git a/security/landlock/access.h b/security/landlock/access.h index bbbb41f41147..f843835851d0 100644 --- a/security/landlock/access.h +++ b/security/landlock/access.h @@ -61,6 +61,10 @@ union access_masks_all { static_assert(sizeof(typeof_member(union access_masks_all, masks)) == sizeof(typeof_member(union access_masks_all, all))); +#define _LANDLOCK_LAYER_MASK_PADDING \ + (BITS_PER_TYPE(access_mask_t) - LANDLOCK_NUM_ACCESS_MAX - \ + IS_ENABLED(CONFIG_SECURITY_LANDLOCK_LOG)) + /** * struct layer_mask - The access rights and rule flags for a layer. * @@ -81,6 +85,10 @@ struct layer_mask { */ access_mask_t quiet : 1; #endif /* CONFIG_SECURITY_LANDLOCK_LOG */ + /** + * @__pad: Padding for the compiler's bitfield initialization. + */ + access_mask_t __pad : _LANDLOCK_LAYER_MASK_PADDING; } __packed __aligned(sizeof(access_mask_t)); /* diff --git a/security/landlock/domain.h b/security/landlock/domain.h index 5ce2f91488d5..caa3d19d2c43 100644 --- a/security/landlock/domain.h +++ b/security/landlock/domain.h @@ -243,7 +243,8 @@ struct landlock_domain { * overlapping access rights. These layers are set once * and never changed for the lifetime of the domain. */ - struct access_masks handled_masks[]; + struct access_masks + handled_masks[] __counted_by(num_layers); }; }; }; diff --git a/security/landlock/fs.c b/security/landlock/fs.c index 330a1871bf94..cab43892ec2f 100644 --- a/security/landlock/fs.c +++ b/security/landlock/fs.c @@ -356,14 +356,14 @@ int landlock_append_fs_rule(struct landlock_ruleset *const ruleset, * held for BTF consistency (enforced by lockdep_assert_held in * TP_fast_assign). */ - if (!err && trace_landlock_add_rule_fs_enabled()) { + if (!err && trace_landlock_add_rule_path_beneath_enabled()) { char *buffer __free(__putname) = __getname(); const char *pathname = buffer ? resolve_path_for_trace(path, buffer) : ""; - trace_landlock_add_rule_fs(ruleset, access_rights, path, - pathname); + trace_landlock_add_rule_path_beneath( + ruleset, flags, access_rights, path, pathname); } mutex_unlock(&ruleset->lock); @@ -423,8 +423,8 @@ static bool unmask_layers_fs(const struct landlock_domain *const domain, ret = landlock_unmask_layers(domain, id, masks, &rule); if (rule) - trace_landlock_check_rule_fs(domain, rule, access_request, - dentry); + trace_landlock_check_rule_inode(domain, rule, access_request, + dentry); return ret; } diff --git a/security/landlock/log.h b/security/landlock/log.h index e0a6e44f3ddd..faa30e26e42a 100644 --- a/security/landlock/log.h +++ b/security/landlock/log.h @@ -15,6 +15,8 @@ struct landlock_cred_security; struct landlock_hierarchy; +struct sockaddr; +struct task_struct; enum landlock_request_type { LANDLOCK_REQUEST_PTRACE = 1, @@ -25,6 +27,31 @@ enum landlock_request_type { LANDLOCK_REQUEST_SCOPE_SIGNAL, }; +struct landlock_blockers { + access_mask_t access; + enum landlock_request_type type; +}; + +#ifdef CONFIG_TRACEPOINTS + +struct landlock_net_trace { + const struct sockaddr *address; + int addrlen; + u16 socket_family; +}; + +struct landlock_ptrace_trace { + u64 tracee_domain_id; + const struct task_struct *tracer; +}; + +struct landlock_signal_trace { + u64 target_domain_id; + int signal; +}; + +#endif /* CONFIG_TRACEPOINTS */ + /* * We should be careful to only use a variable of this type for * landlock_log_denial(). This way, the compiler can remove it entirely if @@ -52,13 +79,24 @@ struct landlock_request { deny_masks_t deny_masks; optional_access_t quiet_optional_accesses; - /* - * Other-party domain ID for a relational (scope/ptrace) denial, or 0 if - * that party is unsandboxed. An ID, not a pointer: the other task can - * replace its credential and free the domain it referenced. Trace path - * only; audit ignores it. - */ - u64 other_domain_id; +#ifdef CONFIG_TRACEPOINTS + union { + /* + * Other-party domain ID for an abstract UNIX socket scope + * denial, or 0 if that party is unsandboxed. Store an ID, not + * a pointer: the other task can replace its credential and free + * the domain it referenced. + */ + u64 other_domain_id; + + /* Synchronous context for a network denial. */ + const struct landlock_net_trace *trace_net; + /* Synchronous context for a ptrace denial. */ + const struct landlock_ptrace_trace *trace_ptrace; + /* Synchronous context for a signal denial. */ + const struct landlock_signal_trace *trace_signal; + }; +#endif /* CONFIG_TRACEPOINTS */ }; #ifdef CONFIG_SECURITY_LANDLOCK_LOG diff --git a/security/landlock/net.c b/security/landlock/net.c index 8f2aaac54b33..6fe0dbde3b78 100644 --- a/security/landlock/net.c +++ b/security/landlock/net.c @@ -47,7 +47,8 @@ int landlock_append_net_rule(struct landlock_ruleset *const ruleset, * TP_fast_assign). */ if (!err) - trace_landlock_add_rule_net(ruleset, access_rights, port); + trace_landlock_add_rule_net_port(ruleset, flags, access_rights, + port); mutex_unlock(&ruleset->lock); return err; @@ -63,7 +64,7 @@ static bool unmask_layers_net(const struct landlock_domain *const domain, ret = landlock_unmask_layers(domain, id, masks, &rule); if (rule) - trace_landlock_check_rule_net( + trace_landlock_check_rule_net_port( domain, rule, access_request, ntohs((__force __be16)id.key.data)); return ret; @@ -92,7 +93,7 @@ static int current_check_access_socket(struct socket *const sock, return 0; /* Checks for minimal header length to safely read sa_family. */ - if (addrlen < offsetofend(typeof(*address), sa_family)) + if (addrlen < (int)offsetofend(typeof(*address), sa_family)) return -EINVAL; /* @@ -144,6 +145,17 @@ static int current_check_access_socket(struct socket *const sock, .audit.u.net = &audit_net, .access = access_request, .layer_masks = &layer_masks, +#ifdef CONFIG_TRACEPOINTS + .trace_net = + &(struct landlock_net_trace){ + .address = + address, + .addrlen = + addrlen, + .socket_family = + sock_family, + }, +#endif /* CONFIG_TRACEPOINTS */ }); return -EACCES; } @@ -275,14 +287,22 @@ static int current_check_access_socket(struct socket *const sock, audit_net.family = address->sa_family; audit_net.sk = sock->sk; - landlock_log_denial(subject, - &(struct landlock_request){ - .type = LANDLOCK_REQUEST_NET_ACCESS, - .audit.type = LSM_AUDIT_DATA_NET, - .audit.u.net = &audit_net, - .access = access_request, - .layer_masks = &layer_masks, - }); + landlock_log_denial( + subject, &(struct landlock_request){ + .type = LANDLOCK_REQUEST_NET_ACCESS, + .audit.type = LSM_AUDIT_DATA_NET, + .audit.u.net = &audit_net, + .access = access_request, + .layer_masks = &layer_masks, +#ifdef CONFIG_TRACEPOINTS + .trace_net = + &(struct landlock_net_trace){ + .address = address, + .addrlen = addrlen, + .socket_family = sock_family, + }, +#endif /* CONFIG_TRACEPOINTS */ + }); return -EACCES; } diff --git a/security/landlock/ruleset.h b/security/landlock/ruleset.h index b536fa0425b7..cf77f1806a95 100644 --- a/security/landlock/ruleset.h +++ b/security/landlock/ruleset.h @@ -171,9 +171,10 @@ struct landlock_ruleset { * @version: Counter incremented on each successful * landlock_add_rule(2), including when it only extends an existing * rule's access rights. Used by tracepoints to correlate a domain with - * the exact ruleset state it was created from. Protected by @lock. + * the exact successful rule history it was created from. Protected by + * @lock. */ - u32 version; + u64 version; /** * @id: Unique identifier for this ruleset, used for tracing. */ diff --git a/security/landlock/task.c b/security/landlock/task.c index 4491ce31ae04..d9eae86fc552 100644 --- a/security/landlock/task.c +++ b/security/landlock/task.c @@ -88,7 +88,9 @@ static int hook_ptrace_access_check(struct task_struct *const child, const unsigned int mode) { const struct landlock_cred_security *parent_subject; +#ifdef CONFIG_TRACEPOINTS u64 tracee_domain_id = 0; +#endif /* CONFIG_TRACEPOINTS */ int err; /* Quick return for non-landlocked tasks. */ @@ -100,10 +102,10 @@ static int hook_ptrace_access_check(struct task_struct *const child, const struct landlock_domain *const child_dom = landlock_get_task_domain(child); err = domain_ptrace(parent_subject->domain, child_dom); -#ifdef CONFIG_SECURITY_LANDLOCK_LOG +#ifdef CONFIG_TRACEPOINTS if (child_dom) tracee_domain_id = child_dom->hierarchy->id; -#endif /* CONFIG_SECURITY_LANDLOCK_LOG */ +#endif /* CONFIG_TRACEPOINTS */ } if (!err) @@ -121,7 +123,12 @@ static int hook_ptrace_access_check(struct task_struct *const child, .u.tsk = child, }, .layer_plus_one = parent_subject->domain->num_layers, - .other_domain_id = tracee_domain_id, +#ifdef CONFIG_TRACEPOINTS + .trace_ptrace = &(struct landlock_ptrace_trace) { + .tracee_domain_id = tracee_domain_id, + .tracer = current, + }, +#endif /* CONFIG_TRACEPOINTS */ }); return err; @@ -142,7 +149,6 @@ static int hook_ptrace_traceme(struct task_struct *const parent) { const struct landlock_cred_security *parent_subject; const struct landlock_domain *child_dom; - u64 tracee_domain_id = 0; int err; child_dom = landlock_get_current_domain(); @@ -154,12 +160,6 @@ static int hook_ptrace_traceme(struct task_struct *const parent) if (!err) return 0; -#ifdef CONFIG_SECURITY_LANDLOCK_LOG - /* The tracee is the current task; its domain is stable here. */ - if (child_dom) - tracee_domain_id = child_dom->hierarchy->id; -#endif /* CONFIG_SECURITY_LANDLOCK_LOG */ - /* * For the ptrace_traceme case, we log the domain which is the cause of * the denial, which means the parent domain instead of the current @@ -174,7 +174,13 @@ static int hook_ptrace_traceme(struct task_struct *const parent) .u.tsk = current, }, .layer_plus_one = parent_subject->domain->num_layers, - .other_domain_id = tracee_domain_id, +#ifdef CONFIG_TRACEPOINTS + .trace_ptrace = &(struct landlock_ptrace_trace) { + /* The current task's domain is stable here. */ + .tracee_domain_id = child_dom ? child_dom->hierarchy->id : 0, + .tracer = parent, + }, +#endif /* CONFIG_TRACEPOINTS */ }); return err; } @@ -250,8 +256,7 @@ static bool domain_is_scoped(const struct landlock_domain *const client, } static bool sock_is_scoped(struct sock *const other, - const struct landlock_domain *const domain, - u64 *const peer_domain_id) + const struct landlock_domain *const domain) { const struct landlock_domain *dom_other; @@ -269,13 +274,23 @@ static bool sock_is_scoped(struct sock *const other, return false; dom_other = landlock_cred(other->sk_socket->file->f_cred)->domain; -#ifdef CONFIG_SECURITY_LANDLOCK_LOG - *peer_domain_id = dom_other ? dom_other->hierarchy->id : 0; -#endif /* CONFIG_SECURITY_LANDLOCK_LOG */ return domain_is_scoped(domain, dom_other, LANDLOCK_SCOPE_ABSTRACT_UNIX_SOCKET); } +#ifdef CONFIG_TRACEPOINTS + +static u64 get_socket_domain_id(const struct sock *const other) +{ + const struct landlock_domain *domain; + + lockdep_assert_held(&unix_sk(other)->lock); + domain = landlock_cred(other->sk_socket->file->f_cred)->domain; + return domain ? domain->hierarchy->id : 0; +} + +#endif /* CONFIG_TRACEPOINTS */ + static bool is_abstract_socket(struct sock *const sock) { struct unix_address *addr = unix_sk(sock)->addr; @@ -299,7 +314,6 @@ static int hook_unix_stream_connect(struct sock *const sock, struct sock *const newsk) { size_t handle_layer; - u64 peer_domain_id = 0; const struct landlock_cred_security *const subject = landlock_get_applicable_subject(current_cred(), unix_scope, &handle_layer); @@ -311,7 +325,7 @@ static int hook_unix_stream_connect(struct sock *const sock, if (!is_abstract_socket(other)) return 0; - if (!sock_is_scoped(other, subject->domain, &peer_domain_id)) + if (!sock_is_scoped(other, subject->domain)) return 0; landlock_log_denial(subject, &(struct landlock_request) { @@ -323,7 +337,9 @@ static int hook_unix_stream_connect(struct sock *const sock, }, }, .layer_plus_one = handle_layer + 1, - .other_domain_id = peer_domain_id, +#ifdef CONFIG_TRACEPOINTS + .other_domain_id = get_socket_domain_id(other), +#endif /* CONFIG_TRACEPOINTS */ }); return -EPERM; } @@ -332,7 +348,6 @@ static int hook_unix_may_send(struct socket *const sock, struct socket *const other) { size_t handle_layer; - u64 peer_domain_id = 0; const struct landlock_cred_security *const subject = landlock_get_applicable_subject(current_cred(), unix_scope, &handle_layer); @@ -350,7 +365,7 @@ static int hook_unix_may_send(struct socket *const sock, if (!is_abstract_socket(other->sk)) return 0; - if (!sock_is_scoped(other->sk, subject->domain, &peer_domain_id)) + if (!sock_is_scoped(other->sk, subject->domain)) return 0; landlock_log_denial(subject, &(struct landlock_request) { @@ -362,7 +377,9 @@ static int hook_unix_may_send(struct socket *const sock, }, }, .layer_plus_one = handle_layer + 1, - .other_domain_id = peer_domain_id, +#ifdef CONFIG_TRACEPOINTS + .other_domain_id = get_socket_domain_id(other->sk), +#endif /* CONFIG_TRACEPOINTS */ }); return -EPERM; } @@ -377,7 +394,9 @@ static int hook_task_kill(struct task_struct *const p, { bool is_scoped; size_t handle_layer; +#ifdef CONFIG_TRACEPOINTS u64 target_domain_id = 0; +#endif /* CONFIG_TRACEPOINTS */ const struct landlock_cred_security *subject; if (!cred) { @@ -409,10 +428,10 @@ static int hook_task_kill(struct task_struct *const p, is_scoped = domain_is_scoped(subject->domain, other, signal_scope.scope); -#ifdef CONFIG_SECURITY_LANDLOCK_LOG +#ifdef CONFIG_TRACEPOINTS if (other) target_domain_id = other->hierarchy->id; -#endif /* CONFIG_SECURITY_LANDLOCK_LOG */ +#endif /* CONFIG_TRACEPOINTS */ } if (!is_scoped) @@ -425,7 +444,12 @@ static int hook_task_kill(struct task_struct *const p, .u.tsk = p, }, .layer_plus_one = handle_layer + 1, - .other_domain_id = target_domain_id, +#ifdef CONFIG_TRACEPOINTS + .trace_signal = &(struct landlock_signal_trace) { + .target_domain_id = target_domain_id, + .signal = sig, + }, +#endif /* CONFIG_TRACEPOINTS */ }); return -EPERM; } @@ -435,7 +459,9 @@ static int hook_file_send_sigiotask(struct task_struct *tsk, { const struct landlock_cred_security *subject; bool is_scoped = false; +#ifdef CONFIG_TRACEPOINTS u64 target_domain_id = 0; +#endif /* CONFIG_TRACEPOINTS */ /* Lock already held by send_sigio() and send_sigurg(). */ lockdep_assert_held(&fown->lock); @@ -468,10 +494,10 @@ static int hook_file_send_sigiotask(struct task_struct *tsk, is_scoped = domain_is_scoped(subject->domain, other, signal_scope.scope); -#ifdef CONFIG_SECURITY_LANDLOCK_LOG +#ifdef CONFIG_TRACEPOINTS if (other) target_domain_id = other->hierarchy->id; -#endif /* CONFIG_SECURITY_LANDLOCK_LOG */ +#endif /* CONFIG_TRACEPOINTS */ } if (!is_scoped) @@ -486,7 +512,12 @@ static int hook_file_send_sigiotask(struct task_struct *tsk, #ifdef CONFIG_SECURITY_LANDLOCK_LOG .layer_plus_one = landlock_file(fown->file)->fown_layer + 1, #endif /* CONFIG_SECURITY_LANDLOCK_LOG */ - .other_domain_id = target_domain_id, +#ifdef CONFIG_TRACEPOINTS + .trace_signal = &(struct landlock_signal_trace) { + .target_domain_id = target_domain_id, + .signal = signum ? signum : SIGIO, + }, +#endif /* CONFIG_TRACEPOINTS */ }); return -EPERM; } diff --git a/security/landlock/trace.c b/security/landlock/trace.c index 8c21e5de6f0d..225dbf37bab0 100644 --- a/security/landlock/trace.c +++ b/security/landlock/trace.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include "access.h" @@ -61,8 +62,8 @@ void landlock_trace_free_domain(const struct landlock_hierarchy *const hierarchy * * @request: Detail of the user space request. * @youngest_denied: The youngest hierarchy node that denied the access. - * @missing: The set of denied access rights. - * @same_exec: Whether the current task is the same executable that called + * @missing: The final missing access subset, when applicable. + * @same_exec: Whether the policy subject is the same executable that called * landlock_restrict_self() for the denying domain, as computed * by landlock_log_denial(). * @logged: Whether the domain's policy selects this denial for logging, as @@ -83,6 +84,10 @@ void landlock_trace_denial( case LANDLOCK_REQUEST_FS_ACCESS: case LANDLOCK_REQUEST_FS_CHANGE_TOPOLOGY: if (trace_landlock_deny_access_fs_enabled()) { + const struct landlock_blockers blockers = { + .access = missing, + .type = request->type, + }; char *buf __free(__putname) = __getname(); struct path dentry_path; const char *pathname; @@ -147,29 +152,67 @@ void landlock_trace_denial( trace_landlock_deny_access_fs(youngest_denied, same_exec, logged, - missing, path, pathname); + &blockers, path, + pathname); } break; case LANDLOCK_REQUEST_NET_ACCESS: - if (trace_landlock_deny_access_net_enabled()) + if (trace_landlock_deny_access_net_enabled()) { + const struct landlock_net_trace *const trace_net = + request->trace_net; + const struct landlock_blockers blockers = { + .access = missing, + .type = request->type, + }; + struct sockaddr_storage address = {}; + + if (WARN_ON_ONCE(!trace_net || !trace_net->address)) + return; + + if (WARN_ON_ONCE( + trace_net->addrlen < + (int)offsetofend(struct sockaddr, + sa_family) || + trace_net->addrlen > (int)sizeof(address))) + return; + + memcpy(&address, trace_net->address, + trace_net->addrlen); trace_landlock_deny_access_net( - youngest_denied, same_exec, logged, missing, + youngest_denied, same_exec, logged, &blockers, request->audit.u.net->sk, - ntohs(request->audit.u.net->sport), - ntohs(request->audit.u.net->dport)); + trace_net->socket_family, &address, + trace_net->addrlen); + } break; case LANDLOCK_REQUEST_PTRACE: - if (trace_landlock_deny_ptrace_enabled()) - trace_landlock_deny_ptrace(youngest_denied, same_exec, - logged, - request->other_domain_id, - request->audit.u.tsk); + if (trace_landlock_deny_ptrace_enabled()) { + const struct landlock_ptrace_trace *const trace_ptrace = + request->trace_ptrace; + + if (WARN_ON_ONCE(!trace_ptrace || + !trace_ptrace->tracer)) + return; + + trace_landlock_deny_ptrace( + youngest_denied, same_exec, logged, + trace_ptrace->tracee_domain_id, + request->audit.u.tsk, trace_ptrace->tracer); + } break; case LANDLOCK_REQUEST_SCOPE_SIGNAL: - if (trace_landlock_deny_scope_signal_enabled()) + if (trace_landlock_deny_scope_signal_enabled()) { + const struct landlock_signal_trace *const trace_signal = + request->trace_signal; + + if (WARN_ON_ONCE(!trace_signal)) + return; + trace_landlock_deny_scope_signal( youngest_denied, same_exec, logged, - request->other_domain_id, request->audit.u.tsk); + trace_signal->target_domain_id, + request->audit.u.tsk, trace_signal->signal); + } break; case LANDLOCK_REQUEST_SCOPE_ABSTRACT_UNIX_SOCKET: if (trace_landlock_deny_scope_abstract_unix_socket_enabled()) diff --git a/sound/core/seq/seq_compat.c b/sound/core/seq/seq_compat.c index 22679dca9aae..80110501da6f 100644 --- a/sound/core/seq/seq_compat.c +++ b/sound/core/seq/seq_compat.c @@ -44,7 +44,9 @@ static int snd_seq_call_port_info_ioctl(struct snd_seq_client *client, unsigned return -EFAULT; data->kernel = NULL; - err = snd_seq_kernel_client_ctl(client->number, cmd, data); + scoped_guard(mutex, &client->ioctl_mutex) { + err = snd_seq_kernel_client_ctl(client->number, cmd, data); + } if (err < 0) return err; diff --git a/sound/hda/codecs/realtek/alc269.c b/sound/hda/codecs/realtek/alc269.c index 5127a111c59c..be5482530aa2 100644 --- a/sound/hda/codecs/realtek/alc269.c +++ b/sound/hda/codecs/realtek/alc269.c @@ -4025,6 +4025,7 @@ enum { ALC269_FIXUP_SONY_VAIO, ALC275_FIXUP_SONY_VAIO_GPIO2, ALC269_FIXUP_DELL_M101Z, + ALC256_FIXUP_ASUS_PM3406CHA, ALC269_FIXUP_SKU_IGNORE, ALC269_FIXUP_ASUS_G73JW, ALC269_FIXUP_ASUS_N7601ZM_PINS, @@ -4124,6 +4125,7 @@ enum { ALC285_FIXUP_HP_SPECTRE_X360_EB1, ALC285_FIXUP_HP_SPECTRE_X360_DF1, ALC285_FIXUP_HP_ENVY_X360, + ALC285_FIXUP_HP_ENVY_13AQ, ALC288_FIXUP_DELL_HEADSET_MODE, ALC288_FIXUP_DELL1_MIC_NO_PRESENCE, ALC288_FIXUP_DELL_XPS_13, @@ -4199,6 +4201,7 @@ enum { ALC225_FIXUP_DELL_WYSE_AIO_MIC_NO_PRESENCE, ALC225_FIXUP_WYSE_AUTO_MUTE, ALC225_FIXUP_WYSE_DISABLE_MIC_VREF, + ALC225_FIXUP_HP_AIO_SPK, ALC286_FIXUP_ACER_AIO_HEADSET_MIC, ALC256_FIXUP_ASUS_HEADSET_MIC, ALC256_FIXUP_ASUS_MIC_NO_PRESENCE, @@ -4384,6 +4387,7 @@ enum { ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN_HEADSET, ALC285_LENOVO_DAC_RENAME, ALC287_FIXUP_YOGA9_SPEAKER2_TO_DAC1, + ALC256_FIXUP_IPASON_SMARTBOOK_S1, }; /* A special fixup for Lenovo C940 and Yoga Duet 7; @@ -4481,6 +4485,14 @@ static const struct hda_fixup alc269_fixups[] = { {} } }, + [ALC256_FIXUP_ASUS_PM3406CHA] = { + .type = HDA_FIXUP_VERBS, + .v.verbs = (const struct hda_verb[]) { + { 0x20, AC_VERB_SET_COEF_INDEX, 0x10 }, + { 0x20, AC_VERB_SET_PROC_COEF, 0x7f20 }, + {} + } + }, [ALC269_FIXUP_SKU_IGNORE] = { .type = HDA_FIXUP_FUNC, .v.func = alc_fixup_sku_ignore, @@ -5719,6 +5731,14 @@ static const struct hda_fixup alc269_fixups[] = { .chained = true, .chain_id = ALC269_FIXUP_HEADSET_MODE_NO_HP_MIC }, + [ALC225_FIXUP_HP_AIO_SPK] = { + .type = HDA_FIXUP_VERBS, + .v.verbs = (const struct hda_verb[]) { + { 0x20, AC_VERB_SET_COEF_INDEX, 0x10 }, + { 0x20, AC_VERB_SET_PROC_COEF, 0x0220 }, + { } + }, + }, [ALC286_FIXUP_ACER_AIO_HEADSET_MIC] = { .type = HDA_FIXUP_VERBS, .v.verbs = (const struct hda_verb[]) { @@ -6400,6 +6420,15 @@ static const struct hda_fixup alc269_fixups[] = { .chained = true, .chain_id = ALC285_FIXUP_HP_GPIO_AMP_INIT, }, + [ALC285_FIXUP_HP_ENVY_13AQ] = { + .type = HDA_FIXUP_PINS, + .v.pins = (const struct hda_pintbl[]) { + { 0x14, 0x90170150 }, /* front speakers */ + { } + }, + .chained = true, + .chain_id = ALC285_FIXUP_HP_GPIO_AMP_INIT, + }, [ALC287_FIXUP_IDEAPAD_BASS_SPK_AMP] = { .type = HDA_FIXUP_FUNC, .v.func = alc285_fixup_ideapad_s740_coef, @@ -7087,6 +7116,8 @@ static const struct hda_fixup alc269_fixups[] = { [ALC236_FIXUP_DELL_HP_POP_NOISE] = { .type = HDA_FIXUP_FUNC, .v.func = alc285_fixup_invalidate_dacs, + .chained = true, + .chain_id = ALC255_FIXUP_DELL1_MIC_NO_PRESENCE }, [ALC274_FIXUP_HP_89E9_GPIO] = { .type = HDA_FIXUP_FUNC, @@ -7136,6 +7167,13 @@ static const struct hda_fixup alc269_fixups[] = { .chained = true, .chain_id = ALC287_FIXUP_TXNW2781_I2C, }, + [ALC256_FIXUP_IPASON_SMARTBOOK_S1] = { + .type = HDA_FIXUP_PINS, + .v.pins = (const struct hda_pintbl[]) { + { 0x1b, 0x90170110 }, /* the real internal speaker */ + { } + }, + }, }; static const struct hda_quirk alc269_fixup_tbl[] = { @@ -7183,6 +7221,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1025, 0x142b, "Acer Swift SF314-42", ALC255_FIXUP_ACER_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1025, 0x1430, "Acer TravelMate B311R-31", ALC256_FIXUP_ACER_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1025, 0x1466, "Acer Aspire A515-56", ALC255_FIXUP_ACER_HEADPHONE_AND_MIC), + SND_PCI_QUIRK(0x1025, 0x151e, "Acer Nitro AN515-45", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1025, 0x1534, "Acer Predator PH315-54", ALC255_FIXUP_ACER_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1025, 0x1539, "Acer Nitro 5 AN515-57", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1025, 0x159c, "Acer Nitro 5 AN515-58", ALC287_FIXUP_ACER_MICMUTE_LED), @@ -7196,8 +7235,10 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1025, 0x1679, "Acer Nitro 16 AN16-41", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1025, 0x169a, "Acer Swift SFG16", ALC256_FIXUP_ACER_SFG16_MICMUTE_LED), SND_PCI_QUIRK(0x1025, 0x171e, "Acer Nitro ANV15-51", ALC245_FIXUP_ACER_MICMUTE_LED), + SND_PCI_QUIRK(0x1025, 0x172c, "Acer Nitro ANV16-41", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1025, 0x1731, "Acer Predator PHN16-72", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1025, 0x173a, "Acer Swift SFG14-73", ALC245_FIXUP_ACER_MICMUTE_LED), + SND_PCI_QUIRK(0x1025, 0x173b, "Acer Swift SFG16-72", ALC245_FIXUP_ACER_MICMUTE_LED), SND_PCI_QUIRK(0x1025, 0x1758, "Acer Nitro ANV15-41", ALC245_FIXUP_ACER_MICMUTE_LED), SND_PCI_QUIRK(0x1025, 0x1826, "Acer Helios ZPC", ALC287_FIXUP_PREDATOR_SPK_CS35L41_I2C_2), SND_PCI_QUIRK(0x1025, 0x182c, "Acer Helios ZPD", ALC287_FIXUP_PREDATOR_SPK_CS35L41_I2C_2), @@ -7303,6 +7344,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1028, 0x164b, "Dell", ALC293_FIXUP_DELL1_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x103c, 0x1586, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC2), SND_PCI_QUIRK(0x103c, 0x18e6, "HP", ALC269_FIXUP_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x2164, "HP Pavilion 15 Notebook PC", ALC269_FIXUP_LIMIT_INT_MIC_BOOST_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x218b, "HP", ALC269_FIXUP_LIMIT_INT_MIC_BOOST_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x21f9, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), SND_PCI_QUIRK(0x103c, 0x2210, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC1), @@ -7385,9 +7427,11 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x854a, "HP EliteBook 830 G6", ALC285_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x854d, "HP EliteBook 840 G6", ALC285_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x856a, "HP Pavilion 15-cs1xxx", ALC295_FIXUP_HP_PAVILION_MUTE_LED_1B), + HDA_CODEC_QUIRK(0x103c, 0x85ba, "HP Pavilion AiO 24", ALC225_FIXUP_HP_AIO_SPK), SND_PCI_QUIRK(0x103c, 0x85c6, "HP Pavilion x360 Convertible 14-dy1xxx", ALC295_FIXUP_HP_MUTE_LED_COEFBIT11), SND_PCI_QUIRK(0x103c, 0x85de, "HP Envy x360 13-ar0xxx", ALC285_FIXUP_HP_ENVY_X360), SND_PCI_QUIRK(0x103c, 0x85f0, "HP Laptop 15-dw0xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), + SND_PCI_QUIRK(0x103c, 0x85f1, "HP Laptop 15-dw1xxx", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_GPIO), SND_PCI_QUIRK(0x103c, 0x85f3, "HP 250 G8 Notebook PC", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x8603, "HP Omen 17-cb0xxx", ALC285_FIXUP_HP_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x860c, "HP ZBook 17 G6", ALC285_FIXUP_HP_GPIO_AMP_INIT), @@ -7395,6 +7439,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x861f, "HP Elite Dragonfly G1", ALC285_FIXUP_HP_GPIO_AMP_INIT), SND_PCI_QUIRK(0x103c, 0x864f, "HP Laptop 15-dy0xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x869d, "HP", ALC236_FIXUP_HP_MUTE_LED), + SND_PCI_QUIRK(0x103c, 0x86ad, "HP ENVY 13-aq1xxx", ALC285_FIXUP_HP_ENVY_13AQ), SND_PCI_QUIRK(0x103c, 0x86c1, "HP Laptop 15-da3001TU", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x86c7, "HP Envy AiO 32", ALC274_FIXUP_HP_ENVY_GPIO), SND_PCI_QUIRK(0x103c, 0x86c8, "HP Laptop 14s-dr1xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), @@ -7423,6 +7468,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8786, "HP OMEN 15", ALC285_FIXUP_HP_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x8787, "HP OMEN 15", ALC285_FIXUP_HP_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x8788, "HP OMEN 15", ALC285_FIXUP_HP_MUTE_LED), + SND_PCI_QUIRK(0x103c, 0x87b2, "HP Pavilion Gaming Laptop 15-ec1xxx", ALC285_FIXUP_HP_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x87b7, "HP Laptop 14-fq0xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x87c8, "HP", ALC287_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x87cb, "HP Pavilion 15-eg0xxx", ALC287_FIXUP_HP_GPIO_LED), @@ -7483,6 +7529,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8975, "HP EliteBook x360 840 Aero G9", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x897d, "HP mt440 Mobile Thin Client U74", ALC236_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8981, "HP Elite Dragonfly G3", ALC245_FIXUP_CS35L41_SPI_4), + SND_PCI_QUIRK(0x103c, 0x8987, "HP Pavilion 14-dv1001TU", ALC287_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x898a, "HP Pavilion 15-eg100", ALC287_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x898e, "HP EliteBook 835 G9", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x898f, "HP EliteBook 835 G9", ALC287_FIXUP_CS35L41_I2C_2), @@ -7520,7 +7567,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8a2d, "HP Envy 16", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x8a2e, "HP Envy 16", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x8a30, "HP Envy 17", ALC287_FIXUP_CS35L41_I2C_2), - SND_PCI_QUIRK(0x103c, 0x8a31, "HP Envy 15", ALC287_FIXUP_CS35L41_I2C_2), + SND_PCI_QUIRK(0x103c, 0x8a31, "HP ENVY x360 2-in-1 Laptop 15-ey0xxx", ALC287_FIXUP_CS35L41_I2C_2_HP_MUTE_LEDS), SND_PCI_QUIRK(0x103c, 0x8a34, "HP Pavilion x360 2-in-1 Laptop 14-ek0xxx", ALC245_FIXUP_HP_MUTE_LED_COEFBIT), SND_PCI_QUIRK(0x103c, 0x8a36, "HP Pavilion Plus 14-eh0xxx", ALC245_FIXUP_HP_MUTE_LED_COEFBIT), SND_PCI_QUIRK(0x103c, 0x8a3d, "HP Victus 15-fb0xxx (MB 8A3D)", ALC245_FIXUP_HP_MUTE_LED_V2_COEFBIT), @@ -7581,6 +7628,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8bb4, "HP Slim OMEN", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x8bb6, "HP Laptop 15-fd0039nt", ALC236_FIXUP_HP_15_FD0XXX), SND_PCI_QUIRK(0x103c, 0x8bbe, "HP Victus 16-r0xxx (MB 8BBE)", ALC245_FIXUP_HP_MUTE_LED_COEFBIT), + SND_PCI_QUIRK(0x103c, 0x8bc7, "HP Pavilion 15-eh3174nw", ALC287_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8bc8, "HP Victus 15-fa1xxx", ALC245_FIXUP_HP_MUTE_LED_COEFBIT), SND_PCI_QUIRK(0x103c, 0x8bcd, "HP Omen 16-xd0xxx", ALC245_FIXUP_HP_MUTE_LED_V1_COEFBIT), SND_PCI_QUIRK(0x103c, 0x8bd4, "HP Victus 16-s0xxx (MB 8BD4)", ALC245_FIXUP_HP_MUTE_LED_COEFBIT), @@ -7603,6 +7651,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8c17, "HP Spectre x360 2-in-1 Laptop 16-aa0xxx", ALC245_FIXUP_HP_SPECTRE_X360_16_AA0XXX), SND_PCI_QUIRK(0x103c, 0x8c21, "HP Pavilion Plus Laptop 14-ey0XXX", ALC245_FIXUP_HP_X360_MUTE_LEDS), SND_PCI_QUIRK(0x103c, 0x8c2d, "HP Victus 15-fa1xxx (MB 8C2D)", ALC245_FIXUP_HP_MUTE_LED_COEFBIT), + SND_PCI_QUIRK(0x103c, 0x8c2f, "HP Victus 15-fb2xxx", ALC245_FIXUP_HP_MUTE_LED_COEFBIT), SND_PCI_QUIRK(0x103c, 0x8c30, "HP Victus 15-fb1xxx", ALC245_FIXUP_HP_MUTE_LED_COEFBIT), SND_PCI_QUIRK(0x103c, 0x8c3f, "HP Victus 15-fa1xxx (MB 8C3F)", ALC245_FIXUP_HP_MUTE_LED_COEFBIT), SND_PCI_QUIRK(0x103c, 0x8c46, "HP EliteBook 830 G11", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED), @@ -7712,6 +7761,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8e37, "HP 16 Piston OmniBook X", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x8e3a, "HP Agusta", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x8e3b, "HP Agusta", ALC287_FIXUP_CS35L41_I2C_2), + SND_PCI_QUIRK(0x103c, 0x8e43, "HP Laptop 15-fd2xxx", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_GPIO), SND_PCI_QUIRK(0x103c, 0x8e60, "HP OmniBook 7 Laptop 16-bh0xxx", ALC245_FIXUP_CS35L41_I2C_2_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x8e61, "HP Trekker ", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x8e62, "HP Trekker ", ALC287_FIXUP_CS35L41_I2C_2), @@ -7814,6 +7864,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1043, 0x1652, "ASUS ROG Zephyrus Do 15 SE", ALC289_FIXUP_ASUS_ZEPHYRUS_DUAL_SPK), SND_PCI_QUIRK(0x1043, 0x1662, "ASUS GV301QH", ALC294_FIXUP_ASUS_DUAL_SPK), SND_PCI_QUIRK(0x1043, 0x1663, "ASUS GU603ZI/ZJ/ZQ/ZU/ZV", ALC285_FIXUP_ASUS_HEADSET_MIC), + SND_PCI_QUIRK(0x1043, 0x1664, "ASUS ExpertBook PM3406CHA", ALC256_FIXUP_ASUS_PM3406CHA), SND_PCI_QUIRK(0x1043, 0x1683, "ASUS UM3402YAR", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x1043, 0x16a3, "ASUS UX3402VA", ALC245_FIXUP_CS35L41_SPI_2), SND_PCI_QUIRK(0x1043, 0x16b2, "ASUS GU603", ALC289_FIXUP_ASUS_GA401), @@ -7916,7 +7967,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1043, 0x3ef0, "ASUS Strix G635LR_LW_LX", ALC287_FIXUP_TXNW2781_I2C), SND_PCI_QUIRK(0x1043, 0x3f00, "ASUS Strix G815LH_LM_LP", ALC287_FIXUP_TXNW2781_I2C), SND_PCI_QUIRK(0x1043, 0x3f10, "ASUS Strix G835LR_LW_LX", ALC287_FIXUP_TXNW2781_I2C), - SND_PCI_QUIRK(0x1043, 0x3f20, "ASUS Strix G615LR_LW", ALC287_FIXUP_TXNW2781_I2C), + SND_PCI_QUIRK(0x1043, 0x3f20, "ASUS Strix G615LR_LW", ALC287_FIXUP_TXNW2781_I2C_ASUS), SND_PCI_QUIRK(0x1043, 0x3f30, "ASUS Strix G815LR_LW", ALC287_FIXUP_TXNW2781_I2C), SND_PCI_QUIRK(0x1043, 0x3fd0, "ASUS B3605CVA", ALC245_FIXUP_CS35L41_SPI_2), SND_PCI_QUIRK(0x1043, 0x3ff0, "ASUS B5405CVA", ALC245_FIXUP_CS35L41_SPI_2), @@ -7939,6 +7990,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x10cf, 0x1757, "Lifebook E752", ALC269_FIXUP_LIFEBOOK_HP_PIN), SND_PCI_QUIRK(0x10cf, 0x1845, "Lifebook U904", ALC269_FIXUP_LIFEBOOK_EXTMIC), SND_PCI_QUIRK(0x10ec, 0x10f2, "Intel Reference board", ALC700_FIXUP_INTEL_REFERENCE), + SND_PCI_QUIRK(0x10ec, 0x111e, "Higole F9B", ALC290_FIXUP_MONO_SPEAKERS), SND_PCI_QUIRK(0x10ec, 0x118c, "Medion EE4254 MD62100", ALC256_FIXUP_MEDION_HEADSET_NO_PRESENCE), SND_PCI_QUIRK(0x10ec, 0x119e, "Positivo SU C1400", ALC269_FIXUP_ASPIRE_HEADSET_MIC), SND_PCI_QUIRK(0x10ec, 0x11bc, "VAIO VJFE-IL", ALC269_FIXUP_LIMIT_INT_MIC_BOOST), @@ -8164,7 +8216,6 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x17aa, 0x3834, "Lenovo IdeaPad Slim 9i 14ITL5", ALC287_FIXUP_YOGA7_14ITL_SPEAKERS), SND_PCI_QUIRK(0x17aa, 0x383d, "Legion Y9000X 2019", ALC285_FIXUP_LEGION_Y9000X_SPEAKERS), SND_PCI_QUIRK(0x17aa, 0x3843, "Lenovo Yoga 9i / Yoga Book 9i", ALC287_FIXUP_LENOVO_YOGA_BOOK_9I), - SND_PCI_QUIRK(0x17aa, 0x3846, "Lenovo Yoga Pro 9 16IAH10", ALC287_FIXUP_YOGA9_SPEAKER2_TO_DAC1), /* Yoga Pro 7 14IMH9 shares PCI SSID 17aa:3847 with Legion 7 16ACHG6; * use codec SSID to distinguish them */ @@ -8189,6 +8240,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { HDA_CODEC_QUIRK(0x17aa, 0x38a8, "Legion Pro 7 16ARX8H", ALC287_FIXUP_TAS2781_I2C), /* this must match before PCI SSID 17aa:386f below */ SND_PCI_QUIRK(0x17aa, 0x386f, "Legion Pro 7i 16IAX7", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x17aa, 0x3870, "Lenovo Yoga 7 14ARB7", ALC287_FIXUP_YOGA7_14ARB7_I2C), + SND_PCI_QUIRK(0x17aa, 0x3871, "Lenovo ThinkBook 16p Gen 3 ARH", ALC287_FIXUP_YOGA7_14ARB7_I2C), SND_PCI_QUIRK(0x17aa, 0x3874, "Legion 7i 16IAX7", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x17aa, 0x3877, "Lenovo Legion 7 Slim 16ARHA7", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x17aa, 0x3878, "Lenovo Legion 7 Slim 16ARHA7", ALC287_FIXUP_CS35L41_I2C_2), @@ -8239,6 +8291,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x17aa, 0x38d5, "Yoga S990-16 Pro IMH YC Quad", ALC287_FIXUP_TAS2781_I2C), SND_PCI_QUIRK(0x17aa, 0x38d6, "Yoga S990-16 Pro IMH VECO Quad", ALC287_FIXUP_TAS2781_I2C), SND_PCI_QUIRK(0x17aa, 0x38d7, "Lenovo Yoga 9 14IMH9", ALC287_FIXUP_YOGA9_14IMH9_BASS_SPK_PIN), + SND_PCI_QUIRK(0x17aa, 0x38dc, "Lenovo ThinkBook G8", ALC285_FIXUP_SPEAKER2_TO_DAC1), SND_PCI_QUIRK(0x17aa, 0x38df, "Yoga Y990 Intel YC Dual", ALC287_FIXUP_TAS2781_I2C), SND_PCI_QUIRK(0x17aa, 0x38e0, "Yoga Y990 Intel VECO Dual", ALC287_FIXUP_TAS2781_I2C), SND_PCI_QUIRK(0x17aa, 0x38f8, "Yoga Book 9i", ALC287_FIXUP_TAS2781_I2C), @@ -8258,7 +8311,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x17aa, 0x3912, "Lenovo Xiaoxin 14 GT", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), SND_PCI_QUIRK(0x17aa, 0x3913, "Lenovo 145", ALC236_FIXUP_LENOVO_INV_DMIC), SND_PCI_QUIRK(0x17aa, 0x391f, "Yoga S990-16 pro Quad YC Quad", ALC287_FIXUP_TXNW2781_I2C), - SND_PCI_QUIRK(0x17aa, 0x3920, "Yoga S990-16 pro Quad VECO Quad", ALC287_FIXUP_TXNW2781_I2C), + SND_PCI_QUIRK(0x17aa, 0x3920, "Yoga S990-16 pro Quad VECO Quad", ALC287_FIXUP_YOGA9_SPEAKER2_TO_DAC1), SND_PCI_QUIRK(0x17aa, 0x3929, "Thinkbook 13x Gen 5", ALC287_FIXUP_MG_RTKC_CSAMP_CS35L41_I2C_THINKPAD), SND_PCI_QUIRK(0x17aa, 0x392b, "Thinkbook 13x Gen 5", ALC287_FIXUP_MG_RTKC_CSAMP_CS35L41_I2C_THINKPAD), HDA_CODEC_QUIRK(0x17aa, 0x3936, "Legion R9000P ADR10H", ALC287_FIXUP_LENOVO_LEGION_AW88399), @@ -8354,6 +8407,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x2014, 0x800a, "Positivo ARN50", ALC269_FIXUP_LIMIT_INT_MIC_BOOST), SND_PCI_QUIRK(0x2039, 0x0001, "Inspur S14-G1", ALC295_FIXUP_CHROME_BOOK), SND_PCI_QUIRK(0x2145, 0x0001, "Star Labs StarFighter", ALC233_FIXUP_STARLABS_STARFIGHTER), + SND_PCI_QUIRK(0x2782, 0x0206, "IPASON SmartBook S1", ALC256_FIXUP_IPASON_SMARTBOOK_S1), SND_PCI_QUIRK(0x2782, 0x0214, "VAIO VJFE-CL", ALC269_FIXUP_LIMIT_INT_MIC_BOOST), SND_PCI_QUIRK(0x2782, 0x0228, "Infinix ZERO BOOK 13", ALC269VB_FIXUP_INFINIX_ZERO_BOOK_13), SND_PCI_QUIRK(0x2782, 0x0232, "CHUWI CoreBook XPro", ALC269VB_FIXUP_CHUWI_COREBOOK_XPRO), diff --git a/sound/hda/codecs/realtek/realtek.c b/sound/hda/codecs/realtek/realtek.c index db365a746b1a..bc5226fad52b 100644 --- a/sound/hda/codecs/realtek/realtek.c +++ b/sound/hda/codecs/realtek/realtek.c @@ -1292,6 +1292,7 @@ static void alc_headset_mode_unplugged(struct hda_codec *codec) } switch (codec->core.vendor_id) { + case 0x10ec0235: case 0x10ec0255: alc_process_coef_fw(codec, coef0255); break; @@ -1407,6 +1408,7 @@ static void alc_headset_mode_mic_in(struct hda_codec *codec, hda_nid_t hp_pin, }; switch (codec->core.vendor_id) { + case 0x10ec0235: case 0x10ec0255: alc_write_coef_idx(codec, 0x45, 0xc489); snd_hda_set_pin_ctl_cache(codec, hp_pin, 0); @@ -1562,6 +1564,7 @@ static void alc_headset_mode_default(struct hda_codec *codec) alc_process_coef_fw(codec, coef0225); alc_hp_enable_unmute(codec, 75); break; + case 0x10ec0235: case 0x10ec0255: alc_process_coef_fw(codec, coef0255); break; @@ -1663,6 +1666,7 @@ static void alc_headset_mode_ctia(struct hda_codec *codec) }; switch (codec->core.vendor_id) { + case 0x10ec0235: case 0x10ec0255: alc_process_coef_fw(codec, coef0255); break; @@ -1780,6 +1784,7 @@ static void alc_headset_mode_omtp(struct hda_codec *codec) }; switch (codec->core.vendor_id) { + case 0x10ec0235: case 0x10ec0255: alc_process_coef_fw(codec, coef0255); break; @@ -1881,6 +1886,7 @@ static void alc_determine_headset_type(struct hda_codec *codec) } switch (codec->core.vendor_id) { + case 0x10ec0235: case 0x10ec0255: alc_process_coef_fw(codec, coef0255); msleep(300); diff --git a/sound/hda/controllers/tegra.c b/sound/hda/controllers/tegra.c index 31c14c4bbe68..674cc0f79c3c 100644 --- a/sound/hda/controllers/tegra.c +++ b/sound/hda/controllers/tegra.c @@ -374,7 +374,8 @@ static int hda_tegra_first_init(struct azx *chip, struct platform_device *pdev) * in powers of 2, next available ratio is 16 which can be * used as a limiting factor here. */ - if (of_device_is_compatible(np, "nvidia,tegra30-hda")) + if (of_device_is_compatible(np, "nvidia,tegra30-hda") || + of_device_is_compatible(np, "nvidia,tegra194-hda")) chip->bus.core.sdo_limit = 16; /* codec detection */ diff --git a/sound/pci/ctxfi/cthw20k2.c b/sound/pci/ctxfi/cthw20k2.c index 36066ffe8784..26a6fa69f5f5 100644 --- a/sound/pci/ctxfi/cthw20k2.c +++ b/sound/pci/ctxfi/cthw20k2.c @@ -1895,6 +1895,7 @@ static int hw_adc_init(struct hw *hw, const struct adc_conf *info) if (hw->model == CTSB1270) { /* Set up the PCM4220 ADC on Titanium HD */ + data = hw_read_20kx(hw, GPIO_DATA); data &= ~0x0C; if (1 == info->msr) data |= 0x00; /* Single Speed Mode 32-50kHz */ @@ -1903,6 +1904,7 @@ static int hw_adc_init(struct hw *hw, const struct adc_conf *info) else data |= 0x04; /* Quad Speed Mode 108kHz-216kHz */ hw_write_20kx(hw, GPIO_DATA, data); + usleep_range(10000, 11000); } hw_adc_start(hw); diff --git a/sound/pci/ice1712/ice1712.c b/sound/pci/ice1712/ice1712.c index d6bfd8fb20e6..1fad8c418132 100644 --- a/sound/pci/ice1712/ice1712.c +++ b/sound/pci/ice1712/ice1712.c @@ -2523,7 +2523,7 @@ static int snd_ice1712_probe(struct pci_dev *pci, const struct pci_device_id *pci_id) { static int dev; - struct snd_card *card __free(snd_card_unref) = NULL; + struct snd_card *card __free(snd_card_free) = NULL; struct snd_ice1712 *ice; int pcm_dev = 0, err; const struct snd_ice1712_card_info * const *tbl, *c; diff --git a/sound/soc/amd/acp-config.c b/sound/soc/amd/acp-config.c index 2d50be8190c1..b3b73096ac74 100644 --- a/sound/soc/amd/acp-config.c +++ b/sound/soc/amd/acp-config.c @@ -51,6 +51,13 @@ static const struct dmi_system_id acp70_acpi_flag_override_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "83V9"), }, }, + { + /* Lenovo Yoga Pro 7 14ASP9 (Strix Point, ACP 7.0) */ + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "83HN"), + }, + }, { .matches = { DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC"), @@ -63,6 +70,13 @@ static const struct dmi_system_id acp70_acpi_flag_override_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "Vivobook 18 M1807GA"), }, }, + { + /* ASUS Zenbook S14 UM5406GA (Strix Point, ACP 7.0) */ + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_PRODUCT_NAME, "Zenbook S14 UM5406GA"), + }, + }, { /* HP OmniBook X Flip 14-kc0xxx (Strix Point, ACP 7.2) */ .matches = { diff --git a/sound/soc/amd/acp/Kconfig b/sound/soc/amd/acp/Kconfig index a04c301df4d6..e880e44b9763 100644 --- a/sound/soc/amd/acp/Kconfig +++ b/sound/soc/amd/acp/Kconfig @@ -181,6 +181,7 @@ config SND_SOC_AMD_LEGACY_SDW_MACH select SND_SOC_RT715_SDW select SND_SOC_RT715_SDCA_SDW select SND_SOC_RT722_SDCA_SDW + select SND_SOC_TAS2783_SDW help This option enables Legacy(No DSP) sound card support for SoundWire enabled AMD platforms along with ACP PDM controller. diff --git a/sound/soc/amd/acp/amd-acp70-acpi-match.c b/sound/soc/amd/acp/amd-acp70-acpi-match.c index 815e088b437d..2a9da6de7bbc 100644 --- a/sound/soc/amd/acp/amd-acp70-acpi-match.c +++ b/sound/soc/amd/acp/amd-acp70-acpi-match.c @@ -748,7 +748,27 @@ static const struct snd_soc_acpi_link_adr acp70_rt721_only[] = { {} }; +static const struct snd_soc_acpi_link_adr acp70_tas2783a_l0_rt712_vb_l1[] = { + { + .mask = BIT(0), + .num_adr = ARRAY_SIZE(tas2783_2_adr), + .adr_d = tas2783_2_adr, + }, + { + .mask = BIT(1), + .num_adr = ARRAY_SIZE(rt712_vb_1_group1_adr), + .adr_d = rt712_vb_1_group1_adr, + }, + {} +}; + struct snd_soc_acpi_mach snd_soc_acpi_amd_acp70_sdw_machines[] = { + { + .link_mask = BIT(0) | BIT(1), + .links = acp70_tas2783a_l0_rt712_vb_l1, + .machine_check = snd_soc_acpi_amd_sdca_is_device_rt712_vb, + .drv_name = "amd_sdw", + }, { .link_mask = BIT(0), .links = acp70_tas2783_2, diff --git a/sound/soc/amd/yc/acp6x-mach.c b/sound/soc/amd/yc/acp6x-mach.c index 385fede6d77f..e798bf6a9cd0 100644 --- a/sound/soc/amd/yc/acp6x-mach.c +++ b/sound/soc/amd/yc/acp6x-mach.c @@ -248,6 +248,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "21J6"), } }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21JN"), + } + }, { .driver_data = &acp6x_card, .matches = { @@ -381,6 +388,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "83Q3"), } }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "83UU"), + } + }, { .driver_data = &acp6x_card, .matches = { diff --git a/sound/soc/codecs/Kconfig b/sound/soc/codecs/Kconfig index d88593c2bac8..8e251e1d84b4 100644 --- a/sound/soc/codecs/Kconfig +++ b/sound/soc/codecs/Kconfig @@ -1989,9 +1989,7 @@ config SND_SOC_RT715_SDCA_SDW config SND_SOC_RT766_SDCA_SDW tristate "Realtek RT766 SDCA Codec - SDW" depends on SOUNDWIRE - depends on SND_SOC_SDCA - select SND_SOC_SDCA_HID - select SND_SOC_SDCA_IRQ + depends on SND_SOC_SDCA_CLASS select REGMAP_SOUNDWIRE select REGMAP_SOUNDWIRE_MBQ diff --git a/sound/soc/codecs/lpass-wsa-macro.c b/sound/soc/codecs/lpass-wsa-macro.c index cfd2ac0a6cda..38e37908baf9 100644 --- a/sound/soc/codecs/lpass-wsa-macro.c +++ b/sound/soc/codecs/lpass-wsa-macro.c @@ -1841,6 +1841,7 @@ static int wsa_macro_enable_interpolator(struct snd_soc_dapm_widget *w, int event) { struct snd_soc_component *component = snd_soc_dapm_to_component(w->dapm); + unsigned int gain; u16 gain_reg; u16 reg; struct wsa_macro *wsa = snd_soc_component_get_drvdata(component); @@ -1882,6 +1883,8 @@ static int wsa_macro_enable_interpolator(struct snd_soc_dapm_widget *w, CDC_WSA_RX_PGA_HALF_DB_MASK, CDC_WSA_RX_PGA_HALF_DB_ENABLE); } + gain = snd_soc_component_read(component, gain_reg); + snd_soc_component_write(component, gain_reg, gain); wsa_macro_config_ear_spkr_gain(component, wsa, event, gain_reg); break; diff --git a/sound/soc/codecs/max98363.c b/sound/soc/codecs/max98363.c index 0e32b7a94cb0..4a6d7b163acf 100644 --- a/sound/soc/codecs/max98363.c +++ b/sound/soc/codecs/max98363.c @@ -200,7 +200,7 @@ static int max98363_sdw_dai_hw_params(struct snd_pcm_substream *substream, struct max98363_priv *max98363 = snd_soc_component_get_drvdata(component); - struct sdw_stream_config stream_config; + struct sdw_stream_config stream_config = {0}; struct sdw_port_config port_config; enum sdw_data_direction direction; struct sdw_stream_runtime *stream; diff --git a/sound/soc/codecs/rt1017-sdca-sdw.c b/sound/soc/codecs/rt1017-sdca-sdw.c index caf75e5657ef..01cc2511a7d0 100644 --- a/sound/soc/codecs/rt1017-sdca-sdw.c +++ b/sound/soc/codecs/rt1017-sdca-sdw.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -573,11 +574,10 @@ static int rt1017_sdca_pcm_hw_params(struct snd_pcm_substream *substream, { struct snd_soc_component *component = dai->component; struct rt1017_sdca_priv *rt1017 = snd_soc_component_get_drvdata(component); - struct sdw_stream_config stream_config; + struct sdw_stream_config stream_config = {0}; struct sdw_port_config port_config; - enum sdw_data_direction direction; struct sdw_stream_runtime *sdw_stream; - int retval, port, num_channels, ch_mask; + int retval, port; unsigned int sampling_rate; dev_dbg(dai->dev, "%s %s", __func__, dai->name); @@ -591,28 +591,17 @@ static int rt1017_sdca_pcm_hw_params(struct snd_pcm_substream *substream, /* SoundWire specific configuration */ /* port 1 for playback */ - if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { - direction = SDW_DATA_DIR_RX; + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) port = 1; - } else { - direction = SDW_DATA_DIR_TX; + else port = 2; - } - num_channels = params_channels(params); - ch_mask = (1 << num_channels) - 1; - - stream_config.frame_rate = params_rate(params); - stream_config.ch_count = num_channels; - stream_config.bps = snd_pcm_format_width(params_format(params)); - stream_config.direction = direction; - - port_config.ch_mask = ch_mask; + snd_sdw_params_to_config(substream, params, &stream_config, &port_config); port_config.num = port; dev_dbg(dai->dev, "frame_rate %d, ch_count %d, bps %d, direction %d, ch_mask %d, port: %d\n", - params_rate(params), num_channels, snd_pcm_format_width(params_format(params)), - direction, ch_mask, port); + stream_config.frame_rate, stream_config.ch_count, stream_config.bps, + stream_config.direction, port_config.ch_mask, port); retval = sdw_stream_add_slave(rt1017->sdw_slave, &stream_config, &port_config, 1, sdw_stream); diff --git a/sound/soc/codecs/rt274.c b/sound/soc/codecs/rt274.c index f8c370106504..3c47f1ae8db9 100644 --- a/sound/soc/codecs/rt274.c +++ b/sound/soc/codecs/rt274.c @@ -619,9 +619,11 @@ static int rt274_hw_params(struct snd_pcm_substream *substream, unsigned int val = 0; int d_len_code = 0, c_len_code = 0; + /* bit 14 Sample Rate Base, 0: 48kHz 1: 44.1kHz */ switch (params_rate(params)) { - /* bit 14 0:48K 1:44.1K */ case 44100: + val |= BIT(14); + break; case 48000: break; default: diff --git a/sound/soc/codecs/rt286.c b/sound/soc/codecs/rt286.c index 4217467904d6..bcd9f5abcda8 100644 --- a/sound/soc/codecs/rt286.c +++ b/sound/soc/codecs/rt286.c @@ -953,7 +953,7 @@ static int rt286_probe(struct snd_soc_component *component) if (rt286->i2c->irq) schedule_delayed_work(&rt286->jack_detect_work, - msecs_to_jiffies(50)); + msecs_to_jiffies(1250)); return 0; } diff --git a/sound/soc/codecs/wm8903.c b/sound/soc/codecs/wm8903.c index 156e1e24a388..fbf388cf966d 100644 --- a/sound/soc/codecs/wm8903.c +++ b/sound/soc/codecs/wm8903.c @@ -676,7 +676,7 @@ SOC_ENUM("DRC Decay Rate", drc_decay), SOC_ENUM("DRC FF Delay", drc_ff_delay), SOC_SINGLE("DRC Anticlip Switch", WM8903_DRC_0, 1, 1, 0), SOC_SINGLE("DRC QR Switch", WM8903_DRC_0, 2, 1, 0), -SOC_SINGLE_TLV("DRC QR Threshold Volume", WM8903_DRC_0, 6, 3, 0, drc_tlv_max), +SOC_SINGLE_TLV("DRC QR Threshold Volume", WM8903_DRC_1, 6, 3, 0, drc_tlv_max), SOC_ENUM("DRC QR Decay Rate", drc_qr_decay), SOC_SINGLE("DRC Smoothing Switch", WM8903_DRC_0, 3, 1, 0), SOC_SINGLE("DRC Smoothing Hysteresis Switch", WM8903_DRC_0, 0, 1, 0), diff --git a/sound/soc/intel/boards/bytcr_rt5651.c b/sound/soc/intel/boards/bytcr_rt5651.c index 62cb4856c797..c9b1053205de 100644 --- a/sound/soc/intel/boards/bytcr_rt5651.c +++ b/sound/soc/intel/boards/bytcr_rt5651.c @@ -410,6 +410,23 @@ static const struct dmi_system_id byt_rt5651_quirk_table[] = { BYT_RT5651_HP_LR_SWAPPED | BYT_RT5651_MONO_SPEAKER), }, + { + /* Chuwi Hi8 (CWI513) with a BIOS using generic DMI strings */ + .callback = byt_rt5651_quirk_cb, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "Hampoo"), + DMI_MATCH(DMI_BOARD_NAME, "Cherry Trail CR"), + /* Above matches are too generic, add bios-date match */ + DMI_MATCH(DMI_BIOS_DATE, "05/21/2016"), + }, + .driver_data = (void *)(BYT_RT5651_MCLK_EN | + BYT_RT5651_JD1_1 | + BYT_RT5651_OVCD_TH_2000UA | + BYT_RT5651_OVCD_SF_1P0 | + BYT_RT5651_IN2_MAP | + BYT_RT5651_HP_LR_SWAPPED | + BYT_RT5651_MONO_SPEAKER), + }, { /* Chuwi Vi8 Plus (CWI519) */ .callback = byt_rt5651_quirk_cb, diff --git a/sound/soc/intel/common/soc-acpi-intel-lnl-match.c b/sound/soc/intel/common/soc-acpi-intel-lnl-match.c index 3b9758f73238..62b8811395cb 100644 --- a/sound/soc/intel/common/soc-acpi-intel-lnl-match.c +++ b/sound/soc/intel/common/soc-acpi-intel-lnl-match.c @@ -745,6 +745,7 @@ struct snd_soc_acpi_mach snd_soc_acpi_intel_lnl_sdw_machines[] = { .links = lnl_cs42l43_l0, .drv_name = "sof_sdw", .sof_tplg_filename = "sof-lnl-cs42l43-l0.tplg", + .machine_check = snd_soc_acpi_intel_force_function_topology, .get_function_tplg_files = sof_sdw_get_tplg_files, }, { diff --git a/sound/soc/intel/common/soc-acpi-intel-sdca-quirks.c b/sound/soc/intel/common/soc-acpi-intel-sdca-quirks.c index 9fdeb6ad96ac..e62ef81214ff 100644 --- a/sound/soc/intel/common/soc-acpi-intel-sdca-quirks.c +++ b/sound/soc/intel/common/soc-acpi-intel-sdca-quirks.c @@ -53,6 +53,22 @@ bool snd_soc_acpi_intel_no_function_topology(void *arg) } EXPORT_SYMBOL_NS(snd_soc_acpi_intel_no_function_topology, "SND_SOC_ACPI_INTEL_SDCA_QUIRKS"); +static const struct dmi_system_id force_function_topology_quirk_table[] = { + { + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "83R0"), + }, + }, + {} +}; + +bool snd_soc_acpi_intel_force_function_topology(void *arg) +{ + return !dmi_check_system(force_function_topology_quirk_table); +} +EXPORT_SYMBOL_NS(snd_soc_acpi_intel_force_function_topology, "SND_SOC_ACPI_INTEL_SDCA_QUIRKS"); + bool snd_soc_acpi_intel_rt712_vb_no_function_topology(void *arg) { return snd_soc_acpi_intel_sdca_is_device_rt712_vb(arg) && diff --git a/sound/soc/intel/common/soc-acpi-intel-sdca-quirks.h b/sound/soc/intel/common/soc-acpi-intel-sdca-quirks.h index 60b665ab5924..10179004c86a 100644 --- a/sound/soc/intel/common/soc-acpi-intel-sdca-quirks.h +++ b/sound/soc/intel/common/soc-acpi-intel-sdca-quirks.h @@ -11,6 +11,7 @@ bool snd_soc_acpi_intel_sdca_is_device_rt712_vb(void *arg); bool snd_soc_acpi_intel_no_function_topology(void *arg); +bool snd_soc_acpi_intel_force_function_topology(void *arg); bool snd_soc_acpi_intel_rt712_vb_no_function_topology(void *arg); #endif diff --git a/sound/soc/qcom/lpass-cpu.c b/sound/soc/qcom/lpass-cpu.c index 242bc16da36d..079177a6d81e 100644 --- a/sound/soc/qcom/lpass-cpu.c +++ b/sound/soc/qcom/lpass-cpu.c @@ -1234,8 +1234,8 @@ int asoc_qcom_lpass_cpu_platform_probe(struct platform_device *pdev) } if (drvdata->mi2s_playback_sd_mode[dai_id] == LPAIF_I2SCTL_MODE_QUAD01) { - variant->dai_driver[dai_id].playback.channels_min = 4; - variant->dai_driver[dai_id].playback.channels_max = 4; + variant->dai_driver[i].playback.channels_min = 4; + variant->dai_driver[i].playback.channels_max = 4; } } diff --git a/sound/soc/sdca/sdca_class_function.c b/sound/soc/sdca/sdca_class_function.c index cc7045dc26e6..610931ae5531 100644 --- a/sound/soc/sdca/sdca_class_function.c +++ b/sound/soc/sdca/sdca_class_function.c @@ -388,20 +388,27 @@ static int class_function_probe(struct auxiliary_device *auxdev, ret = devm_pm_runtime_enable(dev); if (ret) - return ret; + goto err_pm; ret = class_function_boot(drv); if (ret) - return ret; + goto err_pm; ret = devm_snd_soc_register_component(dev, cmp_drv, dais, num_dais); - if (ret) - return dev_err_probe(dev, ret, "failed to register component\n"); + if (ret) { + dev_err_probe(dev, ret, "failed to register component\n"); + goto err_pm; + } pm_runtime_mark_last_busy(dev); pm_runtime_put_autosuspend(dev); return 0; + +err_pm: + pm_runtime_put_sync(dev); + + return ret; } static void class_function_remove(struct auxiliary_device *auxdev) @@ -490,8 +497,6 @@ static int class_function_suspend(struct device *dev) struct class_function_drv *drv = auxiliary_get_drvdata(auxdev); int ret; - drv->suspended = true; - /* Ensure runtime resume runs on resume */ ret = pm_runtime_resume_and_get(dev); if (ret) { @@ -499,6 +504,8 @@ static int class_function_suspend(struct device *dev) return ret; } + drv->suspended = true; + sdca_irq_disable(drv->function, drv->core->irq_info); ret = pm_runtime_force_suspend(dev); diff --git a/sound/soc/tegra/tegra186_asrc.c b/sound/soc/tegra/tegra186_asrc.c index 7f360dfaf8b1..d8ae5d997615 100644 --- a/sound/soc/tegra/tegra186_asrc.c +++ b/sound/soc/tegra/tegra186_asrc.c @@ -828,7 +828,7 @@ static const struct snd_kcontrol_new tegra186_asrc_controls[] = { tegra186_asrc_put_input_threshold), SOC_SINGLE_EXT("Stream6 Input Threshold", - ASRC_STREAM_REG(TEGRA186_ASRC_RX_THRESHOLD, 4), 0, 3, 0, + ASRC_STREAM_REG(TEGRA186_ASRC_RX_THRESHOLD, 5), 0, 3, 0, tegra186_asrc_get_input_threshold, tegra186_asrc_put_input_threshold), diff --git a/sound/usb/caiaq/device.c b/sound/usb/caiaq/device.c index 3d821fde4582..f604846ec6e6 100644 --- a/sound/usb/caiaq/device.c +++ b/sound/usb/caiaq/device.c @@ -562,6 +562,9 @@ static int snd_probe(struct usb_interface *intf, ret = init_card(caiaqdev(card)); if (ret < 0) { dev_err(&usb_dev->dev, "unable to init card! (ret=%d)\n", ret); +#ifdef CONFIG_SND_USB_CAIAQ_INPUT + snd_usb_caiaq_input_disconnect(caiaqdev(card)); +#endif snd_card_free(card); return ret; } diff --git a/sound/usb/line6/playback.c b/sound/usb/line6/playback.c index 7ebaf125f969..aa6ddf8746a0 100644 --- a/sound/usb/line6/playback.c +++ b/sound/usb/line6/playback.c @@ -182,6 +182,13 @@ static int submit_audio_out_urb(struct snd_line6_pcm *line6pcm) fsize *= bytes_per_frame; + if (fsize > line6pcm->max_packet_size_out) { + dev_err(line6pcm->line6->ifcdev, + "playback packet too large: %d > %d\n", + fsize, line6pcm->max_packet_size_out); + return -EMSGSIZE; + } + fout->offset = urb_size; fout->length = fsize; urb_size += fsize; diff --git a/sound/usb/misc/ua101.c b/sound/usb/misc/ua101.c index 860a62a3d74b..58d8a3cdc8b9 100644 --- a/sound/usb/misc/ua101.c +++ b/sound/usb/misc/ua101.c @@ -1007,6 +1007,15 @@ static int detect_usb_format(struct ua101 *ua) } ua->playback.usb_pipe = usb_sndisocpipe(ua->dev, usb_endpoint_num(epd)); ua->playback.max_packet_bytes = usb_endpoint_maxp(epd); + + if (ua->capture.max_packet_bytes / ua->capture.frame_bytes * + ua->playback.frame_bytes > ua->playback.max_packet_bytes) { + dev_err(&ua->dev->dev, + "playback packet size %u too small for %u capture frames\n", + ua->playback.max_packet_bytes, + ua->capture.max_packet_bytes / ua->capture.frame_bytes); + return -ENXIO; + } return 0; } diff --git a/sound/usb/mixer_maps.c b/sound/usb/mixer_maps.c index 41454cb403d0..f8e1f78039e1 100644 --- a/sound/usb/mixer_maps.c +++ b/sound/usb/mixer_maps.c @@ -749,6 +749,10 @@ static const struct usbmix_ctl_map usbmix_ctl_maps[] = { .id = USB_ID(0x0db0, 0xa073), .map = msi_mpg_x570s_carbon_max_wifi_alc4080_map, }, + { /* MSI MAG B850M MORTAR WIFI */ + .id = USB_ID(0x0db0, 0xcc78), + .map = msi_mpg_x570s_carbon_max_wifi_alc4080_map, + }, { /* MSI TRX40 */ .id = USB_ID(0x0db0, 0x543d), .map = trx40_mobo_map, diff --git a/sound/usb/quirks.c b/sound/usb/quirks.c index c5870901a8d4..3afc49565963 100644 --- a/sound/usb/quirks.c +++ b/sound/usb/quirks.c @@ -1702,10 +1702,12 @@ int snd_usb_apply_boot_quirk_once(struct usb_device *dev, switch (id) { case USB_ID(0x07fd, 0x0008): /* MOTU M Series, 1st hardware version */ return snd_usb_motu_m_series_boot_quirk(dev); - case USB_ID(0x1397, 0x1234): /* Behringer CM1A */ - return snd_usb_cm1a_boot_quirk(dev); } + /* Behringer devices may need explicit device descriptor read at boot */ + if (USB_ID_VENDOR(id) == 0x1397) + return snd_usb_cm1a_boot_quirk(dev); + return 0; } @@ -2366,6 +2368,10 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_CTL_MSG_DELAY_1M), DEVICE_FLG(0x0a73, 0x003a, /* Mackie DLZ Creator XS */ QUIRK_FLAG_ALWAYS_SET_RATE), + DEVICE_FLG(0x0b05, 0x1826, /* ASUS SupremeFX Hi-Fi */ + QUIRK_FLAG_DISABLE_AUTOSUSPEND), + DEVICE_FLG(0x0b05, 0x1827, /* ASUS SupremeFX Hi-Fi */ + QUIRK_FLAG_DISABLE_AUTOSUSPEND), DEVICE_FLG(0x0b05, 0x18a6, /* ASUSTek Computer, Inc. */ QUIRK_FLAG_MIXER_CAPTURE_MIN_MUTE), DEVICE_FLG(0x0b0e, 0x0349, /* Jabra 550a */ @@ -2510,8 +2516,6 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_FORCE_IFACE_RESET | QUIRK_FLAG_IFACE_DELAY), DEVICE_FLG(0x262a, 0x9302, /* ddHiFi TC44C */ QUIRK_FLAG_DSD_RAW), - DEVICE_FLG(0x2708, 0x0002, /* Audient iD14 */ - QUIRK_FLAG_IGNORE_CTL_ERROR), DEVICE_FLG(0x2772, 0x0502, /* Musical Fidelity M6s DAC */ 0), /* for avoiding QUIRK_FLAG_DSD_RAW with vendor match */ DEVICE_FLG(0x2912, 0x30c8, /* Audioengine D1 */ @@ -2542,6 +2546,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_IGNORE_CTL_ERROR), DEVICE_FLG(0x3255, 0x0000, /* Luxman D-10X */ QUIRK_FLAG_ITF_USB_DSD_DAC | QUIRK_FLAG_CTL_MSG_DELAY), + DEVICE_FLG(0x32bb, 0x0004, /* HiBy FC4 */ + QUIRK_FLAG_DSD_RAW), DEVICE_FLG(0x3302, 0x17c2, /* TTGK Technology USB-C Audio */ QUIRK_FLAG_FORCE_IFACE_RESET | QUIRK_FLAG_IFACE_DELAY), DEVICE_FLG(0x339b, 0x3a07, /* Synaptics HONOR USB-C HEADSET */ @@ -2608,6 +2614,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_DSD_RAW), VENDOR_FLG(0x2622, /* IAG Limited devices */ QUIRK_FLAG_DSD_RAW), + VENDOR_FLG(0x2708, /* Audient devices */ + QUIRK_FLAG_IGNORE_CTL_ERROR), VENDOR_FLG(0x2772, /* Musical Fidelity devices */ QUIRK_FLAG_DSD_RAW), VENDOR_FLG(0x278b, /* Rotel? */ diff --git a/tools/arch/riscv/include/asm/csr.h b/tools/arch/riscv/include/asm/csr.h index 21d8cee04638..8df64314d613 100644 --- a/tools/arch/riscv/include/asm/csr.h +++ b/tools/arch/riscv/include/asm/csr.h @@ -163,12 +163,24 @@ #define HGATP_MODE_SHIFT HGATP32_MODE_SHIFT #endif -/* VSIP & HVIP relation */ +/* + * VSIP & HVIP relation + * + * The bit positions are same between VSIP and HVIP for interrupt + * numbers 13-63, where there's a shift for the SSI, STI and SEI. + */ #define VSIP_TO_HVIP_SHIFT (IRQ_VS_SOFT - IRQ_S_SOFT) -#define VSIP_VALID_MASK ((_AC(1, UL) << IRQ_S_SOFT) | \ +#define VSIP_BIAS_MASK ((_AC(1, UL) << IRQ_S_SOFT) | \ (_AC(1, UL) << IRQ_S_TIMER) | \ - (_AC(1, UL) << IRQ_S_EXT) | \ - (_AC(1, UL) << IRQ_PMU_OVF)) + (_AC(1, UL) << IRQ_S_EXT)) +#define VSIP_NO_BIAS_MASK (_AC(1, UL) << IRQ_PMU_OVF) +#define VSIP_VALID_MASK (VSIP_BIAS_MASK | VSIP_NO_BIAS_MASK) +#define vsip_to_hvip(_vsip) ((((_vsip) & VSIP_BIAS_MASK) << \ + VSIP_TO_HVIP_SHIFT) | \ + ((_vsip) & VSIP_NO_BIAS_MASK)) +#define hvip_to_vsip(_hvip) ((((_hvip) >> VSIP_TO_HVIP_SHIFT) & \ + VSIP_BIAS_MASK) | \ + ((_hvip) & VSIP_NO_BIAS_MASK)) /* AIA CSR bits */ #define TOPI_IID_SHIFT 16 diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index b749c01742ee..bfa64ae6c94d 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -6206,6 +6206,13 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) return -EINVAL; insn = &prog->insns[insn_idx]; + if (is_ldimm64_insn(insn) && (size_t)insn_idx + 1 >= prog->insns_cnt) { + pr_warn("prog '%s': relo #%d: insn #%d (LDIMM64) is truncated\n", + prog->name, i, insn_idx); + err = -EINVAL; + goto out; + } + err = record_relo_core(prog, rec, insn_idx); if (err) { pr_warn("prog '%s': relo #%d: failed to record relocation: %s\n", diff --git a/tools/lib/bpf/relo_core.c b/tools/lib/bpf/relo_core.c index 8ad2715721cf..2672623a4198 100644 --- a/tools/lib/bpf/relo_core.c +++ b/tools/lib/bpf/relo_core.c @@ -980,23 +980,30 @@ static int bpf_core_calc_relo(const char *prog_name, } /* - * Turn instruction for which CO_RE relocation failed into invalid one with + * Turn instruction for which CO-RE relocation failed into invalid one with * distinct signature. */ -static void bpf_core_poison_insn(const char *prog_name, int relo_idx, - int insn_idx, struct bpf_insn *insn) +static int bpf_core_poison_insn(const char *prog_name, int relo_idx, + struct bpf_insn *insn, int insn_idx) { - pr_debug("prog '%s': relo #%d: substituting insn #%d w/ invalid insn\n", - prog_name, relo_idx, insn_idx); - insn->code = BPF_JMP | BPF_CALL; - insn->dst_reg = 0; - insn->src_reg = 0; - insn->off = 0; - /* if this instruction is reachable (not a dead code), - * verifier will complain with the following message: - * invalid func unknown#195896080 - */ - insn->imm = 195896080; /* => 0xbad2310 => "bad relo" */ + int insn_cnt = is_ldimm64_insn(insn) ? 2 : 1; + int i; + + for (i = 0; i < insn_cnt; i++) { + pr_debug("prog '%s': relo #%d: substituting insn #%d w/ invalid insn\n", + prog_name, relo_idx, insn_idx + i); + insn[i].code = BPF_JMP | BPF_CALL; + insn[i].dst_reg = 0; + insn[i].src_reg = 0; + insn[i].off = 0; + /* + * If this instruction is reachable (not dead code), the verifier + * will complain with "invalid func unknown#195896080". + */ + insn[i].imm = 195896080; /* => 0xbad2310 => "bad relo" */ + } + + return 0; } static int insn_bpf_size_to_bytes(struct bpf_insn *insn) @@ -1047,17 +1054,6 @@ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, class = BPF_CLASS(insn->code); - if (res->poison) { -poison: - /* poison second part of ldimm64 to avoid confusing error from - * verifier about "unknown opcode 00" - */ - if (is_ldimm64_insn(insn)) - bpf_core_poison_insn(prog_name, relo_idx, insn_idx + 1, insn + 1); - bpf_core_poison_insn(prog_name, relo_idx, insn_idx, insn); - return 0; - } - orig_val = res->orig_val; new_val = res->new_val; @@ -1065,7 +1061,9 @@ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, case BPF_ALU: case BPF_ALU64: if (BPF_SRC(insn->code) != BPF_K) - return -EINVAL; + goto bad_insn; + if (res->poison) + return bpf_core_poison_insn(prog_name, relo_idx, insn, insn_idx); if (res->validate && insn->imm != orig_val) { pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %d, exp %llu -> %llu\n", prog_name, relo_idx, @@ -1082,6 +1080,8 @@ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, case BPF_LDX: case BPF_ST: case BPF_STX: + if (res->poison) + return bpf_core_poison_insn(prog_name, relo_idx, insn, insn_idx); if (res->validate && insn->off != orig_val) { pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %d, exp %llu -> %llu\n", prog_name, relo_idx, insn_idx, insn->off, (unsigned long long)orig_val, @@ -1097,7 +1097,7 @@ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) accesses field incorrectly. " "Make sure you are accessing pointers, unsigned integers, or fields of matching type and size.\n", prog_name, relo_idx, insn_idx); - goto poison; + return bpf_core_poison_insn(prog_name, relo_idx, insn, insn_idx); } orig_val = insn->off; @@ -1140,6 +1140,9 @@ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, return -EINVAL; } + if (res->poison) + return bpf_core_poison_insn(prog_name, relo_idx, insn, insn_idx); + imm = (__u32)insn[0].imm | ((__u64)insn[1].imm << 32); if (res->validate && imm != orig_val) { pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDIMM64) value: got %llu, exp %llu -> %llu\n", @@ -1157,6 +1160,7 @@ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, break; } default: +bad_insn: pr_warn("prog '%s': relo #%d: trying to relocate unrecognized insn #%d, code:0x%x, src:0x%x, dst:0x%x, off:0x%x, imm:0x%x\n", prog_name, relo_idx, insn_idx, insn->code, (unsigned)insn->src_reg, (unsigned)insn->dst_reg, (unsigned)insn->off, (unsigned)insn->imm); diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h index 2ddb01a059fd..22f24ebef8a9 100644 --- a/tools/sched_ext/include/scx/common.bpf.h +++ b/tools/sched_ext/include/scx/common.bpf.h @@ -106,7 +106,7 @@ u64 scx_bpf_now(void) __ksym __weak; void scx_bpf_events(struct scx_event_stats *events, size_t events__sz) __ksym __weak; s32 scx_bpf_cpu_to_cid(s32 cpu) __ksym __weak; s32 scx_bpf_cid_to_cpu(s32 cid) __ksym __weak; -void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out) __ksym __weak; +void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out, size_t out__sz) __ksym __weak; void scx_bpf_kick_cid(s32 cid, u64 flags) __ksym __weak; s32 scx_bpf_task_cid(const struct task_struct *p) __ksym __weak; s32 scx_bpf_this_cid(void) __ksym __weak; diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index 67b7c01cae55..2f3653199842 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -961,9 +961,6 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init_task, struct task_struct *p, taskc->highpri = false; taskc->core_sched_seq = 0; cmask_init(&taskc->cpus_allowed, 0, scx_bpf_nr_cids()); - bpf_rcu_read_lock(); - cmask_from_cpumask(&taskc->cpus_allowed, p->cpus_ptr); - bpf_rcu_read_unlock(); v = bpf_task_storage_get(&task_ctx_stor, p, NULL, BPF_LOCAL_STORAGE_GET_F_CREATE); diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index 2d960626750e..273853937c25 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -35,8 +35,11 @@ TARGETS += fchmodat2 TARGETS += filesystems TARGETS += filesystems/binderfs TARGETS += filesystems/epoll +TARGETS += filesystems/eventfd TARGETS += filesystems/failfs TARGETS += filesystems/fat +TARGETS += filesystems/openat2 +TARGETS += filesystems/open_tree_ns TARGETS += filesystems/overlayfs TARGETS += filesystems/statmount TARGETS += filesystems/mount-notify @@ -46,7 +49,7 @@ TARGETS += filesystems/move_mount TARGETS += filesystems/empty_mntns TARGETS += filesystems/fsmount_ns TARGETS += filesystems/fscontext_ns -TARGETS += filesystems/mntns_cleanup +TARGETS += filesystems/xattr TARGETS += firmware TARGETS += fpu TARGETS += ftrace @@ -103,7 +106,6 @@ TARGETS += prctl TARGETS += proc TARGETS += pstore TARGETS += ptrace -TARGETS += openat2 TARGETS += rdma TARGETS += resctrl TARGETS += riscv diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_ma_ttrace.c b/tools/testing/selftests/bpf/prog_tests/bpf_ma_ttrace.c new file mode 100644 index 000000000000..a1d41b109940 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/bpf_ma_ttrace.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include "bpf_ma_ttrace.skel.h" + +#define NR_ELEMS 4096 + +/* + * The first free_bulk() starts RCU tasks trace GP. The rest of the elements are + * deleted while it's in flight. They should be freed without further alloc or + * free from this map. + */ +void test_bpf_ma_ttrace(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct bpf_ma_ttrace *skel; + __u32 cnt = NR_ELEMS; + long *vals = NULL; + int *keys = NULL; + int i, err, fd, nr_cpus; + + skel = bpf_ma_ttrace__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + nr_cpus = libbpf_num_possible_cpus(); + if (!ASSERT_GT(nr_cpus, 0, "nr_cpus")) + goto out; + skel->bss->nr_cpus = nr_cpus; + + keys = calloc(NR_ELEMS, sizeof(*keys)); + vals = calloc(NR_ELEMS, sizeof(*vals)); + if (!ASSERT_OK_PTR(keys, "keys") || !ASSERT_OK_PTR(vals, "vals")) + goto out; + for (i = 0; i < NR_ELEMS; i++) + keys[i] = i; + + fd = bpf_map__fd(skel->maps.htab); + err = bpf_map_update_batch(fd, keys, vals, &cnt, NULL); + if (!ASSERT_OK(err, "update_batch") || !ASSERT_EQ(cnt, NR_ELEMS, "update_cnt")) + goto out; + err = bpf_map_delete_batch(fd, keys, &cnt, NULL); + if (!ASSERT_OK(err, "delete_batch") || !ASSERT_EQ(cnt, NR_ELEMS, "delete_cnt")) + goto out; + + /* Wait for all __free_rcu() callbacks to finish */ + for (i = 0; i < 300; i++) { + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.check_ttrace), &opts); + if (!ASSERT_OK(err, "test_run") || !ASSERT_OK(opts.retval, "retval")) + goto out; + if (!skel->bss->in_progress) + break; + usleep(100000); + } + ASSERT_EQ(skel->bss->nr_caches, nr_cpus, "nr_caches"); + ASSERT_EQ(skel->bss->in_progress, 0, "in_progress"); + ASSERT_EQ(skel->bss->not_freed, 0, "not_freed"); +out: + free(keys); + free(vals); + bpf_ma_ttrace__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_mod_race.c b/tools/testing/selftests/bpf/prog_tests/bpf_mod_race.c index ecc3d47919ad..f8497e764beb 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_mod_race.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_mod_race.c @@ -55,38 +55,6 @@ static void *load_module_thread(void *p) return p; } -static int sys_userfaultfd(int flags) -{ - return syscall(__NR_userfaultfd, flags); -} - -static int test_setup_uffd(void *fault_addr) -{ - struct uffdio_register uffd_register = {}; - struct uffdio_api uffd_api = {}; - int uffd; - - uffd = sys_userfaultfd(O_CLOEXEC); - if (uffd < 0) - return -errno; - - uffd_api.api = UFFD_API; - uffd_api.features = 0; - if (ioctl(uffd, UFFDIO_API, &uffd_api)) { - close(uffd); - return -1; - } - - uffd_register.range.start = (unsigned long)fault_addr; - uffd_register.range.len = getpagesize(); - uffd_register.mode = UFFDIO_REGISTER_MODE_MISSING; - if (ioctl(uffd, UFFDIO_REGISTER, &uffd_register)) { - close(uffd); - return -1; - } - return uffd; -} - static void test_bpf_mod_race_config(const struct test_config *config) { void *fault_addr, *skel_fail; @@ -117,7 +85,7 @@ static void test_bpf_mod_race_config(const struct test_config *config) if (!ASSERT_OK(bpf_mod_race__attach(skel), "bpf_mod_kfunc_race__attach")) goto end_destroy; - uffd = test_setup_uffd(fault_addr); + uffd = uffd_block_page(fault_addr); if (!ASSERT_GE(uffd, 0, "userfaultfd open + register address")) goto end_destroy; diff --git a/tools/testing/selftests/bpf/prog_tests/cb_refs.c b/tools/testing/selftests/bpf/prog_tests/cb_refs.c index 78566b817fd7..490e15e7126d 100644 --- a/tools/testing/selftests/bpf/prog_tests/cb_refs.c +++ b/tools/testing/selftests/bpf/prog_tests/cb_refs.c @@ -13,7 +13,7 @@ struct { } cb_refs_tests[] = { { "underflow_prog", "release kfunc bpf_kfunc_call_test_release expects referenced PTR_TO_BTF_ID passed to R1" }, { "leak_prog", "Possibly NULL pointer passed to helper R2" }, - { "nested_cb", "Unreleased reference id=4 alloc_insn=2" }, /* alloc_insn=2{4,5} */ + { "nested_cb", "Unreleased reference id=5 alloc_insn=2" }, /* alloc_insn=2{4,5} */ { "non_cb_transfer_ref", "Unreleased reference id=4 alloc_insn=1" }, /* alloc_insn=1{1,2} */ }; diff --git a/tools/testing/selftests/bpf/prog_tests/core_reloc_raw.c b/tools/testing/selftests/bpf/prog_tests/core_reloc_raw.c index a18d3680fb16..51f42b02a267 100644 --- a/tools/testing/selftests/bpf/prog_tests/core_reloc_raw.c +++ b/tools/testing/selftests/bpf/prog_tests/core_reloc_raw.c @@ -14,6 +14,197 @@ static char log[16 * 1024]; +static int load_core_relo_insns(int btf_fd, struct bpf_insn *insns, int insn_cnt, + struct bpf_func_info *funcs, int func_cnt, + int enum_id, int access_str_off, int insn_idx, + bool relocate) +{ + struct bpf_core_relo relo = { + .insn_off = insn_idx * sizeof(struct bpf_insn), + .type_id = enum_id, + .access_str_off = access_str_off, + .kind = BPF_CORE_ENUMVAL_VALUE, + }; + union bpf_attr attr = { + .prog_type = BPF_PROG_TYPE_SOCKET_FILTER, + .insn_cnt = insn_cnt, + .insns = (__u64)insns, + .license = (__u64)"GPL", + .log_buf = (__u64)log, + .log_size = sizeof(log), + .log_level = 2, + .prog_btf_fd = btf_fd, + .func_info_rec_size = sizeof(struct bpf_func_info), + .func_info = (__u64)funcs, + .func_info_cnt = func_cnt, + }; + + if (relocate) { + attr.core_relo_cnt = 1; + attr.core_relos = (__u64)&relo; + attr.core_relo_rec_size = sizeof(relo); + } + memset(log, 0, sizeof(log)); + return sys_bpf_prog_load(&attr, sizeof(attr), 1); +} + +static void test_early_core_relo(void) +{ + static const char unrecognized[] = "trying to relocate unrecognized insn #2"; + static const struct { + const char *name; + struct bpf_insn insns[2]; + const char *err_msg; + } tests[] = { + { "poison_exit", { BPF_EXIT_INSN() }, unrecognized }, + { "poison_ja", { BPF_JMP_A(1) }, unrecognized }, + { "poison_jmp", { BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 1) }, unrecognized }, + { "poison_jmp32", { BPF_JMP32_IMM(BPF_JEQ, BPF_REG_0, 0, 1) }, unrecognized }, + { "poison_call", { BPF_EMIT_CALL(BPF_FUNC_get_prandom_u32) }, unrecognized }, + { "poison_alu_reg", { BPF_MOV32_REG(BPF_REG_0, BPF_REG_1) }, unrecognized }, + { "poison_alu64_reg", { BPF_MOV64_REG(BPF_REG_0, BPF_REG_1) }, unrecognized }, + { "poison_ld_abs", { BPF_LD_ABS(BPF_W, 0) }, + "insn #2 (LDIMM64) has unexpected form" }, + { "poison_alu_imm", { BPF_MOV32_IMM(BPF_REG_0, 0) } }, + { "poison_alu64_imm", { BPF_MOV64_IMM(BPF_REG_0, 0) } }, + { "poison_ldx", { BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1, 0) } }, + { "poison_st", { BPF_ST_MEM(BPF_W, BPF_REG_10, -4, 0) } }, + { "poison_stx", { BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_0, -4) } }, + { "poison_ldimm64", { BPF_LD_IMM64(BPF_REG_0, 0) } }, + }; + struct test_btf { + struct btf_header hdr; + __u32 types[18]; + char strings[64]; + } raw_btf = { + .hdr = { + .magic = BTF_MAGIC, + .version = BTF_VERSION, + .hdr_len = sizeof(struct btf_header), + .type_off = 0, + .type_len = sizeof(raw_btf.types), + .str_off = offsetof(struct test_btf, strings) - + offsetof(struct test_btf, types), + .str_len = sizeof(raw_btf.strings), + }, + .types = { + BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] int */ + BTF_FUNC_PROTO_ENC(1, 0), /* [2] int (*)(void) */ + BTF_FUNC_ENC(5, 2), /* [3] main_fn */ + BTF_FUNC_ENC(13, 2), /* [4] sub_fn */ + BTF_TYPE_ENC(20, BTF_INFO_ENC(BTF_KIND_ENUM, 0, 1), 4), /* [5] enum */ + BTF_ENUM_ENC(45, 0), /* value = 0 */ + }, + .strings = "\0int\0main_fn\0sub_fn\0core_relo_poison_missing\0value\0" "0", + }; + struct bpf_func_info funcs[] = { + { .insn_off = 0, .type_id = 3 }, + { .insn_off = 3, .type_id = 4 }, + }; + struct bpf_insn core_only[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 1), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + struct bpf_insn subprog[] = { + BPF_CALL_REL(2), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + struct bpf_insn truncated_ldimm64[] = { + BPF_RAW_INSN(BPF_LD | BPF_IMM | BPF_DW, 0, 0, 0, 0), + }; + int access_str_off = 51; /* offset of "0" */ + int enum_id = 5; + int btf_fd, prog_fd = -1, i; + + btf_fd = bpf_btf_load(&raw_btf, sizeof(raw_btf), NULL); + if (!ASSERT_GE(btf_fd, 0, "btf_load")) + goto cleanup; + + if (test__start_subtest("without_func_info")) { + prog_fd = load_core_relo_insns(btf_fd, core_only, ARRAY_SIZE(core_only), NULL, 0, + enum_id, access_str_off, 2, false); + if (!ASSERT_GE(prog_fd, 0, "control_load")) + goto cleanup; + close(prog_fd); + prog_fd = load_core_relo_insns(btf_fd, core_only, ARRAY_SIZE(core_only), NULL, 0, + enum_id, access_str_off, 2, true); + if (!ASSERT_GE(prog_fd, 0, "poisoned_load")) + goto cleanup; + ASSERT_HAS_SUBSTR(log, "substituting insn #2", "poison_log"); + close(prog_fd); + prog_fd = -1; + } + + if (test__start_subtest("before_subprog_validation")) { + prog_fd = load_core_relo_insns(btf_fd, subprog, ARRAY_SIZE(subprog), funcs, 2, + enum_id, access_str_off, 2, true); + if (!ASSERT_LT(prog_fd, 0, "poisoned_load")) + goto cleanup; + ASSERT_HAS_SUBSTR(log, "substituting insn #2", "poison_log"); + ASSERT_HAS_SUBSTR(log, "last insn is not an exit or jmp", "poisoned_load_log"); + } + + if (test__start_subtest("truncated_ldimm64")) { + prog_fd = load_core_relo_insns(btf_fd, truncated_ldimm64, + ARRAY_SIZE(truncated_ldimm64), NULL, 0, + enum_id, access_str_off, 0, true); + if (!ASSERT_LT(prog_fd, 0, "truncated_load")) + goto cleanup; + ASSERT_HAS_SUBSTR(log, "invalid bpf_ld_imm64 insn", "truncated_load_log"); + } + + for (i = 0; i < ARRAY_SIZE(tests); i++) { + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 1), + tests[i].insns[0], + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + bool is_ldimm64 = insns[2].code == (BPF_LD | BPF_DW | BPF_IMM); + + if (!test__start_subtest(tests[i].name)) + continue; + if (is_ldimm64) { + insns[1].off = 2; + insns[3] = tests[i].insns[1]; + } + prog_fd = load_core_relo_insns(btf_fd, insns, ARRAY_SIZE(insns), funcs, 1, + enum_id, access_str_off, 2, false); + if (!ASSERT_GE(prog_fd, 0, "control_load")) + goto cleanup; + close(prog_fd); + prog_fd = load_core_relo_insns(btf_fd, insns, ARRAY_SIZE(insns), funcs, 1, + enum_id, access_str_off, 2, true); + if (!tests[i].err_msg) { + ASSERT_GE(prog_fd, 0, "dead_poison_load"); + ASSERT_HAS_SUBSTR(log, "substituting insn #2", "poison_log"); + if (is_ldimm64) + ASSERT_HAS_SUBSTR(log, "substituting insn #3", "poison_ldimm64_log"); + } else { + ASSERT_LT(prog_fd, 0, "invalid_poison_load"); + ASSERT_HAS_SUBSTR(log, tests[i].err_msg, "invalid_poison_log"); + ASSERT_NULL(strstr(log, "substituting insn"), "invalid_poison_substitution"); + } + close(prog_fd); + prog_fd = -1; + } + +cleanup: + if (env.verbosity > VERBOSE_NORMAL && log[0]) { + printf("-------- program load log start --------\n"); + printf("%s", log); + printf("-------- program load log end ----------\n"); + } + close(prog_fd); + close(btf_fd); +} + /* Check that verifier rejects BPF program containing relocation * pointing to non-existent BTF type. */ @@ -120,6 +311,7 @@ static void test_bad_local_id(void) void test_core_reloc_raw(void) { + test_early_core_relo(); if (test__start_subtest("bad_local_id")) test_bad_local_id(); } diff --git a/tools/testing/selftests/bpf/prog_tests/dynptr.c b/tools/testing/selftests/bpf/prog_tests/dynptr.c index 5fda11590708..4396560365e8 100644 --- a/tools/testing/selftests/bpf/prog_tests/dynptr.c +++ b/tools/testing/selftests/bpf/prog_tests/dynptr.c @@ -9,6 +9,7 @@ enum test_setup_type { SETUP_SYSCALL_SLEEP, SETUP_SKB_PROG, + SETUP_SKB_PROG_NONLINEAR, SETUP_SKB_PROG_TP, SETUP_XDP_PROG, }; @@ -32,6 +33,7 @@ static struct { {"test_ringbuf", SETUP_SYSCALL_SLEEP}, {"test_skb_readonly", SETUP_SKB_PROG}, {"test_dynptr_skb_data", SETUP_SKB_PROG}, + {"test_dynptr_skb_slice_non_linear", SETUP_SKB_PROG_NONLINEAR}, {"test_dynptr_skb_meta_data", SETUP_SKB_PROG}, {"test_dynptr_skb_meta_flags", SETUP_SKB_PROG}, {"test_adjust", SETUP_SYSCALL_SLEEP}, @@ -94,7 +96,9 @@ static void verify_success(const char *prog_name, enum test_setup_type setup_typ bpf_link__destroy(link); break; case SETUP_SKB_PROG: + case SETUP_SKB_PROG_NONLINEAR: { + struct __sk_buff ctx = {}; int prog_fd; char buf[64]; @@ -106,6 +110,12 @@ static void verify_success(const char *prog_name, enum test_setup_type setup_typ .repeat = 1, ); + if (setup_type == SETUP_SKB_PROG_NONLINEAR) { + ctx.data_end = ETH_HLEN + sizeof(struct iphdr); + topts.ctx_in = &ctx; + topts.ctx_size_in = sizeof(ctx); + } + prog_fd = bpf_program__fd(prog); if (!ASSERT_GE(prog_fd, 0, "prog_fd")) goto cleanup; diff --git a/tools/testing/selftests/bpf/prog_tests/exceptions.c b/tools/testing/selftests/bpf/prog_tests/exceptions.c index 3588d6f97fd4..639866ce09a9 100644 --- a/tools/testing/selftests/bpf/prog_tests/exceptions.c +++ b/tools/testing/selftests/bpf/prog_tests/exceptions.c @@ -55,6 +55,7 @@ static void test_exceptions_success(void) RUN_SUCCESS(exception_ext, 0); RUN_SUCCESS(exception_ext_mod_cb_runtime, 35); RUN_SUCCESS(exception_throw_subprog, 1); + RUN_SUCCESS(exception_throw_subprog_stack_cb, 0x1234); RUN_SUCCESS(exception_assert_nz_gfunc, 1); RUN_SUCCESS(exception_assert_zero_gfunc, 1); RUN_SUCCESS(exception_assert_neg_gfunc, 1); diff --git a/tools/testing/selftests/bpf/prog_tests/linked_list.c b/tools/testing/selftests/bpf/prog_tests/linked_list.c index c3d133c6a00d..52fabbee3dd5 100644 --- a/tools/testing/selftests/bpf/prog_tests/linked_list.c +++ b/tools/testing/selftests/bpf/prog_tests/linked_list.c @@ -714,7 +714,7 @@ static void test_btf(void) break; err = btf__load_into_kernel(btf); - ASSERT_EQ(err, -ELOOP, "check btf"); + ASSERT_EQ(err, 0, "check btf"); btf__free(btf); break; } @@ -773,7 +773,7 @@ static void test_btf(void) break; err = btf__load_into_kernel(btf); - ASSERT_EQ(err, -ELOOP, "check btf"); + ASSERT_EQ(err, 0, "check btf"); btf__free(btf); break; } diff --git a/tools/testing/selftests/bpf/prog_tests/local_kptr_ownership.c b/tools/testing/selftests/bpf/prog_tests/local_kptr_ownership.c new file mode 100644 index 000000000000..a487aa68f2ee --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/local_kptr_ownership.c @@ -0,0 +1,296 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include + +#define SPIN_LOCK 2 +#define LIST_HEAD 3 +#define LIST_NODE 4 +/* Keep in sync with BTF_MAX_OWNERSHIP_DEPTH. */ +#define MAX_OWNERSHIP_DEPTH 8 + +static struct btf *init_btf(void) +{ + struct btf *btf; + int id; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf__new_empty")) + return NULL; + id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + if (!ASSERT_EQ(id, 1, "btf__add_int")) + goto err_out; + id = btf__add_struct(btf, "bpf_spin_lock", 4); + if (!ASSERT_EQ(id, SPIN_LOCK, "btf__add_struct bpf_spin_lock")) + goto err_out; + id = btf__add_struct(btf, "bpf_list_head", 16); + if (!ASSERT_EQ(id, LIST_HEAD, "btf__add_struct bpf_list_head")) + goto err_out; + id = btf__add_struct(btf, "bpf_list_node", 24); + if (!ASSERT_EQ(id, LIST_NODE, "btf__add_struct bpf_list_node")) + goto err_out; + return btf; + +err_out: + btf__free(btf); + return NULL; +} + +static int add_local_kptr(struct btf *btf, int pointee_id, const char *tag) +{ + int id; + + id = btf__add_type_tag(btf, tag, pointee_id); + if (!ASSERT_GT(id, 0, "btf__add_type_tag")) + return id; + id = btf__add_ptr(btf, id); + ASSERT_GT(id, 0, "btf__add_ptr"); + return id; +} + +static void test_self_cycle(const char *tag, int expected_err) +{ + struct btf *btf; + int id, err; + + btf = init_btf(); + if (!ASSERT_OK_PTR(btf, "init_btf")) + return; + id = add_local_kptr(btf, 7, tag); + if (id <= 0) + goto out; + id = btf__add_struct(btf, "self_cycle", 8); + if (!ASSERT_EQ(id, 7, "btf__add_struct self_cycle")) + goto out; + err = btf__add_field(btf, "next", 6, 0, 0); + if (!ASSERT_OK(err, "btf__add_field self_cycle::next")) + goto out; + + err = btf__load_into_kernel(btf); + ASSERT_EQ(err, expected_err, "check btf"); +out: + btf__free(btf); +} + +static void test_aba_cycle(void) +{ + struct btf *btf; + int id, err; + + btf = init_btf(); + if (!ASSERT_OK_PTR(btf, "init_btf")) + return; + id = add_local_kptr(btf, 10, "kptr"); + if (id <= 0) + goto out; + id = add_local_kptr(btf, 9, "kptr"); + if (id <= 0) + goto out; + id = btf__add_struct(btf, "cycle_a", 8); + if (!ASSERT_EQ(id, 9, "btf__add_struct cycle_a")) + goto out; + err = btf__add_field(btf, "b", 6, 0, 0); + if (!ASSERT_OK(err, "btf__add_field cycle_a::b")) + goto out; + id = btf__add_struct(btf, "cycle_b", 8); + if (!ASSERT_EQ(id, 10, "btf__add_struct cycle_b")) + goto out; + err = btf__add_field(btf, "a", 8, 0, 0); + if (!ASSERT_OK(err, "btf__add_field cycle_b::a")) + goto out; + + err = btf__load_into_kernel(btf); + ASSERT_EQ(err, -ELOOP, "check btf"); +out: + btf__free(btf); +} + +static void test_mixed_cycle(void) +{ + struct btf *btf; + int id, err; + + btf = init_btf(); + if (!ASSERT_OK_PTR(btf, "init_btf")) + return; + id = add_local_kptr(btf, 7, "kptr"); + if (id <= 0) + goto out; + id = btf__add_struct(btf, "mixed_owner", 20); + if (!ASSERT_EQ(id, 7, "btf__add_struct mixed_owner")) + goto out; + err = btf__add_field(btf, "root", LIST_HEAD, 0, 0); + if (!ASSERT_OK(err, "btf__add_field mixed_owner::root")) + goto out; + err = btf__add_field(btf, "lock", SPIN_LOCK, 128, 0); + if (!ASSERT_OK(err, "btf__add_field mixed_owner::lock")) + goto out; + id = btf__add_decl_tag(btf, "contains:mixed_node:node", 7, 0); + if (!ASSERT_EQ(id, 8, "btf__add_decl_tag mixed_owner")) + goto out; + id = btf__add_struct(btf, "mixed_node", 32); + if (!ASSERT_EQ(id, 9, "btf__add_struct mixed_node")) + goto out; + err = btf__add_field(btf, "node", LIST_NODE, 0, 0); + if (!ASSERT_OK(err, "btf__add_field mixed_node::node")) + goto out; + err = btf__add_field(btf, "owner", 6, 192, 0); + if (!ASSERT_OK(err, "btf__add_field mixed_node::owner")) + goto out; + + err = btf__load_into_kernel(btf); + ASSERT_EQ(err, -ELOOP, "check btf"); +out: + btf__free(btf); +} + +static void test_acyclic_depth(int depth, bool child_first, bool shared_suffix, int expected_err) +{ + int ptr_id[MAX_OWNERSHIP_DEPTH + 1]; + int first_struct_id; + struct btf *btf; + int id, err, i, n, pointee_id; + + btf = init_btf(); + if (!ASSERT_OK_PTR(btf, "init_btf")) + return; + first_struct_id = 5 + 2 * depth; + for (i = 0; i < depth; i++) { + if (i == depth - 1) + pointee_id = first_struct_id + depth; + else + pointee_id = first_struct_id + (child_first ? depth - 2 - i : i + 1); + ptr_id[i] = add_local_kptr(btf, pointee_id, "kptr"); + if (ptr_id[i] <= 0) + goto out; + } + for (n = 0; n < depth; n++) { + char name[32]; + int offset = 0; + + i = child_first ? depth - 1 - n : n; + snprintf(name, sizeof(name), "owner_%d", i); + id = btf__add_struct(btf, name, shared_suffix && !i ? 16 : 8); + if (!ASSERT_EQ(id, first_struct_id + n, "btf__add_struct owner")) + goto out; + if (shared_suffix && !i) { + /* + * Visit the shared suffix through the shorter path before + * reaching it again with less remaining depth. + */ + err = btf__add_field(btf, "suffix", ptr_id[1], 0, 0); + if (!ASSERT_OK(err, "btf__add_field owner::suffix")) + goto out; + offset = 64; + } + err = btf__add_field(btf, "next", ptr_id[i], offset, 0); + if (!ASSERT_OK(err, "btf__add_field owner::next")) + goto out; + } + id = btf__add_struct(btf, "plain_leaf", 4); + if (!ASSERT_EQ(id, first_struct_id + depth, "btf__add_struct plain_leaf")) + goto out; + + err = btf__load_into_kernel(btf); + ASSERT_EQ(err, expected_err, "check btf"); +out: + btf__free(btf); +} + +static void test_graph_depth(bool rbtree, int depth, int expected_err) +{ + int root_type = LIST_HEAD, node_type = LIST_NODE, node_size = 24; + int id, err, i, lock_off, root_off, size; + struct btf *btf; + + btf = init_btf(); + if (!ASSERT_OK_PTR(btf, "init_btf")) + return; + if (rbtree) { + root_type = btf__add_struct(btf, "bpf_rb_root", 16); + if (!ASSERT_GT(root_type, 0, "btf__add_struct bpf_rb_root")) + goto out; + node_type = btf__add_struct(btf, "bpf_rb_node", 32); + if (!ASSERT_GT(node_type, 0, "btf__add_struct bpf_rb_node")) + goto out; + node_size = 32; + } + + for (i = 0; i < depth; i++) { + char name[32], tag[64]; + + lock_off = i ? node_size : 0; + root_off = lock_off + 8; + size = i == depth - 1 ? node_size : root_off + 16; + snprintf(name, sizeof(name), "graph_owner_%d", i); + id = btf__add_struct(btf, name, size); + if (!ASSERT_GT(id, 0, "btf__add_struct graph_owner")) + goto out; + if (i) { + err = btf__add_field(btf, "node", node_type, 0, 0); + if (!ASSERT_OK(err, "btf__add_field graph_owner::node")) + goto out; + } + if (i == depth - 1) + continue; + err = btf__add_field(btf, "lock", SPIN_LOCK, lock_off * 8, 0); + if (!ASSERT_OK(err, "btf__add_field graph_owner::lock")) + goto out; + err = btf__add_field(btf, "root", root_type, root_off * 8, 0); + if (!ASSERT_OK(err, "btf__add_field graph_owner::root")) + goto out; + snprintf(tag, sizeof(tag), "contains:graph_owner_%d:node", i + 1); + err = btf__add_decl_tag(btf, tag, id, i ? 2 : 1); + if (!ASSERT_GT(err, 0, "btf__add_decl_tag graph_owner")) + goto out; + } + + err = btf__load_into_kernel(btf); + ASSERT_EQ(err, expected_err, "check btf"); +out: + btf__free(btf); +} + +void test_local_kptr_ownership(void) +{ + if (test__start_subtest("self_cycle")) + test_self_cycle("kptr", -ELOOP); + if (test__start_subtest("untrusted_self_cycle")) + test_self_cycle("kptr_untrusted", 0); + if (test__start_subtest("percpu_self_cycle")) + test_self_cycle("percpu_kptr", -ELOOP); + if (test__start_subtest("ABA_cycle")) + test_aba_cycle(); + if (test__start_subtest("mixed_graph_root_cycle")) + test_mixed_cycle(); + if (test__start_subtest("max_acyclic")) + test_acyclic_depth(MAX_OWNERSHIP_DEPTH, false, false, 0); + if (test__start_subtest("too_deep_acyclic")) + test_acyclic_depth(MAX_OWNERSHIP_DEPTH + 1, false, false, -ELOOP); + if (test__start_subtest("max_acyclic_child_first")) + test_acyclic_depth(MAX_OWNERSHIP_DEPTH, true, false, 0); + if (test__start_subtest("too_deep_acyclic_child_first")) + test_acyclic_depth(MAX_OWNERSHIP_DEPTH + 1, true, false, -ELOOP); + if (test__start_subtest("max_acyclic_shared_suffix")) + test_acyclic_depth(MAX_OWNERSHIP_DEPTH, false, true, 0); + if (test__start_subtest("too_deep_acyclic_shared_suffix")) + test_acyclic_depth(MAX_OWNERSHIP_DEPTH + 1, false, true, -ELOOP); + if (test__start_subtest("list_three_types")) + test_graph_depth(false, 3, 0); + if (test__start_subtest("list_four_types")) + test_graph_depth(false, 4, 0); + if (test__start_subtest("list_max_depth")) + test_graph_depth(false, MAX_OWNERSHIP_DEPTH, 0); + if (test__start_subtest("list_too_deep")) + test_graph_depth(false, MAX_OWNERSHIP_DEPTH + 1, -ELOOP); + if (test__start_subtest("rbtree_three_types")) + test_graph_depth(true, 3, 0); + if (test__start_subtest("rbtree_four_types")) + test_graph_depth(true, 4, 0); + if (test__start_subtest("rbtree_max_depth")) + test_graph_depth(true, MAX_OWNERSHIP_DEPTH, 0); + if (test__start_subtest("rbtree_too_deep")) + test_graph_depth(true, MAX_OWNERSHIP_DEPTH + 1, -ELOOP); +} diff --git a/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c b/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c index a72ae0b29f6e..7b4a1e24363b 100644 --- a/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c +++ b/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c @@ -1,4 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +#define _GNU_SOURCE +#include #include #include "cgroup_helpers.h" #include "percpu_alloc_array.skel.h" @@ -350,6 +352,103 @@ static void test_lru_percpu_hash_cpu_flag(void) test_percpu_map_cpu_flag(BPF_MAP_TYPE_LRU_PERCPU_HASH); } +/* + * A BPF_F_CPU update that creates an element must zero the value on the other + * cpus, rather than leave them holding whatever the recycled element last + * contained. max_entries is 1 so the second key can only reuse the element + * the first one released. + */ +static void test_percpu_map_cpu_flag_create(enum bpf_map_type map_type, __u32 map_flags) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = map_flags); + const u32 stale = 0xDEADC0DE, fresh = 0xC0FFEE; + int nr_cpus, cpu, map_fd, err, key; + int pinned_cpu, value_cpu; + cpu_set_t old_mask, new_mask; + bool restore_mask = false; + u32 value; + u64 flags; + + nr_cpus = libbpf_num_possible_cpus(); + if (!ASSERT_GT(nr_cpus, 0, "libbpf_num_possible_cpus")) + return; + + if (nr_cpus < 2) { + test__skip(); + return; + } + + map_fd = bpf_map_create(map_type, "cpu_flag_create", sizeof(key), sizeof(value), 1, &opts); + if (!ASSERT_GE(map_fd, 0, "bpf_map_create")) + return; + + /* NO_PREALLOC recycles per cpu, so keep the delete and the create on one cpu. */ + err = sched_getaffinity(0, sizeof(old_mask), &old_mask); + if (!ASSERT_OK(err, "sched_getaffinity")) + goto out; + + pinned_cpu = sched_getcpu(); + if (!ASSERT_GE(pinned_cpu, 0, "sched_getcpu")) + goto out; + + CPU_ZERO(&new_mask); + CPU_SET(pinned_cpu, &new_mask); + err = sched_setaffinity(0, sizeof(new_mask), &new_mask); + if (!ASSERT_OK(err, "sched_setaffinity")) + goto out; + restore_mask = true; + + value_cpu = pinned_cpu ? 0 : 1; + + key = 1; + value = stale; + err = bpf_map_update_elem(map_fd, &key, &value, BPF_F_ALL_CPUS); + if (!ASSERT_OK(err, "bpf_map_update_elem all_cpus")) + goto out; + + err = bpf_map_delete_elem(map_fd, &key); + if (!ASSERT_OK(err, "bpf_map_delete_elem")) + goto out; + + key = 2; + value = fresh; + flags = (u64)value_cpu << 32 | BPF_F_CPU; + err = bpf_map_update_elem(map_fd, &key, &value, flags); + if (!ASSERT_OK(err, "bpf_map_update_elem specified cpu")) + goto out; + + for (cpu = 0; cpu < nr_cpus; cpu++) { + value = 0; + flags = (u64)cpu << 32 | BPF_F_CPU; + err = bpf_map_lookup_elem_flags(map_fd, &key, &value, flags); + if (!ASSERT_OK(err, "bpf_map_lookup_elem_flags specified cpu")) + goto out; + if (!ASSERT_EQ(value, cpu == value_cpu ? fresh : 0, "value on specified cpu")) + goto out; + } + +out: + if (restore_mask) + sched_setaffinity(0, sizeof(old_mask), &old_mask); + close(map_fd); +} + +static void test_percpu_hash_cpu_flag_create(void) +{ + test_percpu_map_cpu_flag_create(BPF_MAP_TYPE_PERCPU_HASH, 0); +} + +static void test_percpu_hash_cpu_flag_create_malloc(void) +{ + test_percpu_map_cpu_flag_create(BPF_MAP_TYPE_PERCPU_HASH, BPF_F_NO_PREALLOC); +} + +static void test_lru_percpu_hash_cpu_flag_create(void) +{ + /* lru without prealloc is -ENOTSUPP, so there is no malloc variant */ + test_percpu_map_cpu_flag_create(BPF_MAP_TYPE_LRU_PERCPU_HASH, 0); +} + static void test_percpu_cgroup_storage_cpu_flag(void) { struct percpu_alloc_array *skel = NULL; @@ -454,6 +553,12 @@ void test_percpu_alloc(void) test_percpu_hash_cpu_flag(); if (test__start_subtest("cpu_flag_lru_percpu_hash")) test_lru_percpu_hash_cpu_flag(); + if (test__start_subtest("cpu_flag_create_percpu_hash")) + test_percpu_hash_cpu_flag_create(); + if (test__start_subtest("cpu_flag_create_percpu_hash_malloc")) + test_percpu_hash_cpu_flag_create_malloc(); + if (test__start_subtest("cpu_flag_create_lru_percpu_hash")) + test_lru_percpu_hash_cpu_flag_create(); if (test__start_subtest("cpu_flag_percpu_cgroup_storage")) test_percpu_cgroup_storage_cpu_flag(); if (test__start_subtest("cpu_flag_array")) diff --git a/tools/testing/selftests/bpf/prog_tests/sock_destroy.c b/tools/testing/selftests/bpf/prog_tests/sock_destroy.c index 9c11938fe597..78d642a02bdb 100644 --- a/tools/testing/selftests/bpf/prog_tests/sock_destroy.c +++ b/tools/testing/selftests/bpf/prog_tests/sock_destroy.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include @@ -110,6 +111,122 @@ static void test_tcp_server(struct sock_destroy_prog *skel) close(serv); } +static void test_tcp_listen_pending(struct sock_destroy_prog *skel) +{ + int serv = -1, clien = -1, accept_serv = -1, n, serv_port; + struct pollfd pfd = { .events = POLLIN }; + char buf[1]; + + serv = start_server(AF_INET6, SOCK_STREAM, NULL, 0, 0); + if (!ASSERT_GE(serv, 0, "start_server")) + goto cleanup; + serv_port = get_socket_local_port(serv); + if (!ASSERT_GE(serv_port, 0, "get_sock_local_port")) + goto cleanup; + skel->bss->serv_port = (__be16)serv_port; + + /* + * Connect but never accept, so the child sits in the accept queue + * of the listener. Wait until it's actually there. + */ + clien = connect_to_fd(serv, 0); + if (!ASSERT_GE(clien, 0, "connect_to_fd")) + goto cleanup; + pfd.fd = serv; + if (!ASSERT_EQ(poll(&pfd, 1, -1), 1, "poll listener")) + goto cleanup; + + /* Run iterator program that destroys server sockets. */ + start_iter_sockets(skel->progs.iter_tcp6_server); + + accept_serv = accept(serv, NULL, NULL); + if (!ASSERT_LT(accept_serv, 0, "accept on destroyed listener")) + goto cleanup; + ASSERT_EQ(errno, EINVAL, "error code on destroyed listener"); + + /* The unaccepted child was reset along with the listener. */ + n = recv(clien, buf, sizeof(buf), 0); + if (!ASSERT_LT(n, 0, "client recv on reset child")) + goto cleanup; + ASSERT_EQ(errno, ECONNRESET, "error code on reset child"); + +cleanup: + if (clien != -1) + close(clien); + if (accept_serv != -1) + close(accept_serv); + if (serv != -1) + close(serv); +} + +static void test_tcp_timewait(struct sock_destroy_prog *skel) +{ + int serv = -1, clien = -1, accept_serv = -1, n; + struct timeval tv = {}; + char buf[1]; + + serv = start_server(AF_INET6, SOCK_STREAM, NULL, 0, 0); + if (!ASSERT_GE(serv, 0, "start_server")) + goto cleanup; + + clien = connect_to_fd(serv, 0); + if (!ASSERT_GE(clien, 0, "connect_to_fd")) + goto cleanup; + + accept_serv = accept(serv, NULL, NULL); + if (!ASSERT_GE(accept_serv, 0, "serv accept")) + goto cleanup; + + /* + * Active close from the client, then close the server side. Once + * recv() sees EOF the server FIN has been processed and the client + * sock is in TIME_WAIT. Block without timeout so a loaded CI box + * can't race us. + */ + if (!ASSERT_OK(setsockopt(clien, SOL_SOCKET, SO_RCVTIMEO, &tv, + sizeof(tv)), "clear rcvtimeo")) + goto cleanup; + if (!ASSERT_OK(shutdown(clien, SHUT_WR), "client shutdown")) + goto cleanup; + + /* + * Make sure the server has seen the client FIN before it closes, + * so the two FINs never cross. + */ + n = recv(accept_serv, buf, sizeof(buf), 0); + if (!ASSERT_EQ(n, 0, "server recv EOF")) + goto cleanup; + + close(accept_serv); + accept_serv = -1; + + /* block until return EOF */ + n = recv(clien, buf, sizeof(buf), 0); + if (!ASSERT_EQ(n, 0, "client recv EOF")) + goto cleanup; + + /* Run iterator program that destroys the timewait client sock. */ + skel->bss->tw_found = 0; + start_iter_sockets(skel->progs.iter_tcp6_timewait); + if (!ASSERT_EQ(skel->bss->tw_found, 1, "timewait sock found")) + goto cleanup; + + ASSERT_OK(skel->bss->tw_destroy_err, "destroy timewait sock"); + + /* The destroyed timewait sock must be gone. */ + skel->bss->tw_found = 0; + start_iter_sockets(skel->progs.iter_tcp6_timewait); + ASSERT_EQ(skel->bss->tw_found, 0, "timewait sock destroyed"); + +cleanup: + if (clien != -1) + close(clien); + if (accept_serv != -1) + close(accept_serv); + if (serv != -1) + close(serv); +} + static void test_udp_client(struct sock_destroy_prog *skel) { int serv = -1, clien = -1, n = 0; @@ -204,6 +321,10 @@ void test_sock_destroy(void) test_tcp_client(skel); if (test__start_subtest("tcp_server")) test_tcp_server(skel); + if (test__start_subtest("tcp_listen_pending")) + test_tcp_listen_pending(skel); + if (test__start_subtest("tcp_timewait")) + test_tcp_timewait(skel); if (test__start_subtest("udp_client")) test_udp_client(skel); if (test__start_subtest("udp_server")) diff --git a/tools/testing/selftests/bpf/prog_tests/spin_lock.c b/tools/testing/selftests/bpf/prog_tests/spin_lock.c index 5c3579438427..e368370262c8 100644 --- a/tools/testing/selftests/bpf/prog_tests/spin_lock.c +++ b/tools/testing/selftests/bpf/prog_tests/spin_lock.c @@ -54,6 +54,8 @@ static struct { { "lock_global_sleepable_helper_subprog", "global function calls are not allowed while holding a lock" }, { "lock_global_sleepable_kfunc_subprog", "global function calls are not allowed while holding a lock" }, { "lock_global_sleepable_subprog_indirect", "global function calls are not allowed while holding a lock" }, + { "callback_value_lock_identity", "bpf_spin_unlock of different lock" }, + { "callback_inner_map_value_lock_identity", "bpf_spin_unlock of different lock" }, }; static int match_regex(const char *pattern, const char *string) diff --git a/tools/testing/selftests/bpf/prog_tests/tc_change_tail_pmtu.c b/tools/testing/selftests/bpf/prog_tests/tc_change_tail_pmtu.c new file mode 100644 index 000000000000..7acdbd5757a9 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/tc_change_tail_pmtu.c @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#include "test_progs.h" +#include "network_helpers.h" +#include "test_tc_change_tail_pmtu.skel.h" + +#define CLIENT_NS "tc-change-tail-cli-ns" +#define SERVER_NS "tc-change-tail-srv-ns" +#define CLIENT_IP "192.168.1.1" +#define SERVER_IP "192.168.1.2" + +#define TEST_PMTU 1000 +#define TEST_MSS_MAX (TEST_PMTU - 20 - 20) +#define TIMEOUT_MS 3000 +#define XFER_BYTES 8192 + +void test_tc_change_tail_pmtu(void) +{ + LIBBPF_OPTS(bpf_tcx_opts, tcx_opts); + int mss_before = 0, mss_after = 0, ifindex, port; + int srv_fd = -1, srv_conn_fd = -1, cli_fd = -1; + struct test_tc_change_tail_pmtu *skel = NULL; + struct nstoken *nstoken = NULL; + static char buf[XFER_BYTES]; + socklen_t optlen; + ssize_t bytes; + size_t total; + + if (!ASSERT_OK(make_netns(CLIENT_NS), "make client ns")) + return; + if (!ASSERT_OK(make_netns(SERVER_NS), "make server ns")) + goto out_client_ns; + + nstoken = open_netns(CLIENT_NS); + if (!ASSERT_OK_PTR(nstoken, "open client ns")) + goto out; + SYS(out, "ip link add veth1 type veth peer name veth2 netns " SERVER_NS); + SYS(out, "ip -4 addr add " CLIENT_IP "/24 dev veth1"); + SYS(out, "ip link set veth1 up"); + ifindex = if_nametoindex("veth1"); + if (!ASSERT_NEQ(ifindex, 0, "if_nametoindex")) + goto out; + close_netns(nstoken); + nstoken = NULL; + + nstoken = open_netns(SERVER_NS); + if (!ASSERT_OK_PTR(nstoken, "open server ns")) + goto out; + SYS(out, "ip -4 addr add " SERVER_IP "/24 dev veth2"); + SYS(out, "ip link set veth2 up"); + srv_fd = start_server(AF_INET, SOCK_STREAM, SERVER_IP, 0, TIMEOUT_MS); + if (!ASSERT_OK_FD(srv_fd, "start server")) + goto out; + close_netns(nstoken); + nstoken = NULL; + + skel = test_tc_change_tail_pmtu__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open and load skeleton")) + goto out; + + port = get_socket_local_port(srv_fd); + if (!ASSERT_GE(port, 0, "get server port")) + goto out; + + skel->bss->server_port = port; + skel->bss->pmtu = TEST_PMTU; + + nstoken = open_netns(CLIENT_NS); + if (!ASSERT_OK_PTR(nstoken, "open client ns")) + goto out; + + skel->links.change_tail_icmp = + bpf_program__attach_tcx(skel->progs.change_tail_icmp, ifindex, + &tcx_opts); + if (!ASSERT_OK_PTR(skel->links.change_tail_icmp, "attach tcx")) + goto out; + + cli_fd = connect_to_fd(srv_fd, TIMEOUT_MS); + if (!ASSERT_OK_FD(cli_fd, "connect to server")) + goto out; + srv_conn_fd = accept(srv_fd, NULL, NULL); + if (!ASSERT_OK_FD(srv_conn_fd, "accept connection")) + goto out; + if (!ASSERT_OK(settimeo(srv_conn_fd, TIMEOUT_MS), "set server timeout")) + goto out; + + optlen = sizeof(mss_before); + if (!ASSERT_OK(getsockopt(cli_fd, IPPROTO_TCP, TCP_MAXSEG, &mss_before, + &optlen), "get mss before")) + goto out; + + bytes = send(cli_fd, buf, sizeof(buf), 0); + if (!ASSERT_EQ(bytes, (ssize_t)sizeof(buf), "send data")) + goto out; + + for (total = 0; total < sizeof(buf); total += bytes) { + bytes = recv(srv_conn_fd, buf, sizeof(buf), 0); + if (bytes <= 0) + break; + } + + ASSERT_EQ(total, sizeof(buf), "receive data"); + ASSERT_OK(skel->data->change_tail_ret, "change tail"); + ASSERT_OK(skel->bss->adjust_room_ret, "adjust room"); + ASSERT_TRUE(skel->bss->icmp_sent, "icmp sent"); + + optlen = sizeof(mss_after); + if (!ASSERT_OK(getsockopt(cli_fd, IPPROTO_TCP, TCP_MAXSEG, &mss_after, + &optlen), "get mss after")) + goto out; + + ASSERT_LT(mss_after, mss_before, "mss reduced"); + ASSERT_LE(mss_after, TEST_MSS_MAX, "mss below pmtu"); +out: + close(srv_conn_fd); + close(cli_fd); + close(srv_fd); + test_tc_change_tail_pmtu__destroy(skel); + close_netns(nstoken); + remove_netns(SERVER_NS); +out_client_ns: + remove_netns(CLIENT_NS); +} diff --git a/tools/testing/selftests/bpf/prog_tests/tramp_prog_detach.c b/tools/testing/selftests/bpf/prog_tests/tramp_prog_detach.c new file mode 100644 index 000000000000..1eb0d7237605 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/tramp_prog_detach.c @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include "tramp_prog_detach.skel.h" +#include "testing_helpers.h" + +/* + * Detach and free progs while a task sleeps in the prog that runs before them + * in the same trampoline image, then let that task continue through the + * image. It must not call into the freed progs. + * + * The task is held in a sleepable prog with userfaultfd, like bpf_mod_race + * does. + */ + +static struct bpf_program *pick_prog(struct tramp_prog_detach *skel, + bool fexit, bool sleepable) +{ + if (fexit) + return sleepable ? skel->progs.fexit_sleepable : + skel->progs.fexit_victim; + return sleepable ? skel->progs.fentry_sleepable : + skel->progs.fentry_victim; +} + +static struct bpf_program *sleepable_prog; + +static void *run_sleepable(void *arg) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + + /* calls bpf_fentry_test1() */ + return (void *)(long)bpf_prog_test_run_opts(bpf_program__fd(sleepable_prog), + &topts); +} + +static struct tramp_prog_detach *load_one(bool fexit, bool sleepable) +{ + struct tramp_prog_detach *skel; + int err; + + skel = tramp_prog_detach__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return NULL; + + bpf_program__set_autoload(pick_prog(skel, fexit, sleepable), true); + err = tramp_prog_detach__load(skel); + if (!ASSERT_OK(err, "load")) + goto err; + skel->bss->pid = getpid(); + err = tramp_prog_detach__attach(skel); + if (!ASSERT_OK(err, "attach")) + goto err; + return skel; +err: + tramp_prog_detach__destroy(skel); + return NULL; +} + +/* The .bss map of a destroyed skeleton goes away when its prog is freed */ +static bool wait_for_map_free(__u32 map_id) +{ + int i, fd; + + for (i = 0; i < 100; i++) { + fd = bpf_map_get_fd_by_id(map_id); + if (fd < 0) + return true; + close(fd); + usleep(100 * 1000); + } + return false; +} + +static void test_detach(bool sleepable_fexit, bool victim_fexit) +{ + struct tramp_prog_detach *sleepable = NULL, *victims[2] = {}; + struct pollfd pfd = { .events = POLLIN }; + struct uffdio_copy uffd_copy = {}; + struct bpf_map_info map_info = {}; + __u32 map_info_len = sizeof(map_info); + struct uffd_msg uffd_msg; + void *fault_page, *src_page = MAP_FAILED; + long page_size = getpagesize(); + bool started = false; + void *thread_ret; + pthread_t thread; + int i, uffd = -1; + + fault_page = mmap(NULL, page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (!ASSERT_NEQ(fault_page, MAP_FAILED, "mmap fault_page")) + return; + src_page = mmap(NULL, page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (!ASSERT_NEQ(src_page, MAP_FAILED, "mmap src_page")) + goto out; + + /* The most recently attached prog runs first */ + for (i = 0; i < ARRAY_SIZE(victims); i++) { + victims[i] = load_one(victim_fexit, false); + if (!victims[i]) + goto out; + } + sleepable = load_one(sleepable_fexit, true); + if (!sleepable) + goto out; + sleepable_prog = pick_prog(sleepable, sleepable_fexit, true); + + /* Not armed yet so this doesn't block, make sure sleepable runs first */ + if (!ASSERT_OK((long)run_sleepable(NULL), "dry run")) + goto out; + for (i = 0; i < ARRAY_SIZE(victims); i++) + if (!ASSERT_LT(sleepable->bss->ts, victims[i]->bss->ts, "prog order")) + goto out; + + uffd = uffd_block_page(fault_page); + if (!ASSERT_GE(uffd, 0, "userfaultfd open + register address")) + goto out; + sleepable->bss->fault_addr = fault_page; + + if (!ASSERT_OK(pthread_create(&thread, NULL, run_sleepable, NULL), + "pthread_create")) + goto out; + started = true; + + /* Wait for the thread to sleep in bpf_copy_from_user() */ + pfd.fd = uffd; + if (!ASSERT_EQ(poll(&pfd, 1, 10000), 1, "poll uffd")) + goto out; + if (!ASSERT_EQ(read(uffd, &uffd_msg, sizeof(uffd_msg)), sizeof(uffd_msg), + "read uffd")) + goto out; + if (!ASSERT_EQ(uffd_msg.event, UFFD_EVENT_PAGEFAULT, "uffd pagefault")) + goto out; + + /* + * Detach and unload the victim progs and wait for them to be freed. + * After the first one, the task is in an image that isn't the + * trampoline's current one anymore. + */ + for (i = 0; i < ARRAY_SIZE(victims); i++) { + if (!ASSERT_OK(bpf_map_get_info_by_fd(bpf_map__fd(victims[i]->maps.bss), + &map_info, &map_info_len), + "victim bss info")) + goto out; + tramp_prog_detach__destroy(victims[i]); + victims[i] = NULL; + if (!ASSERT_TRUE(wait_for_map_free(map_info.id), "victim freed")) + goto out; + } + +out: + /* Let the thread proceed with the rest of the trampoline */ + if (uffd >= 0) { + uffd_copy.dst = (unsigned long)fault_page; + uffd_copy.src = (unsigned long)src_page; + uffd_copy.len = page_size; + ASSERT_OK(ioctl(uffd, UFFDIO_COPY, &uffd_copy), "uffd copy"); + close(uffd); + } + if (started && + ASSERT_OK(pthread_join(thread, &thread_ret), "pthread_join")) + ASSERT_NULL(thread_ret, "blocking run"); + + for (i = 0; i < ARRAY_SIZE(victims); i++) + tramp_prog_detach__destroy(victims[i]); + tramp_prog_detach__destroy(sleepable); + if (src_page != MAP_FAILED) + munmap(src_page, page_size); + munmap(fault_page, page_size); +} + +void serial_test_tramp_prog_detach(void) +{ + /* a task sleeping before the original function is called */ + if (test__start_subtest("fentry")) + test_detach(false, false); + /* a task sleeping after the original function returned */ + if (test__start_subtest("fexit")) + test_detach(true, true); + /* the original function runs in between and must still be called */ + if (test__start_subtest("fentry_fexit")) + test_detach(false, true); +} diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index 64ac49ad67e6..8b439e194bcc 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -23,6 +23,7 @@ #include "verifier_bpf_trap.skel.h" #include "verifier_bswap.skel.h" #include "verifier_btf_ctx_access.skel.h" +#include "verifier_btf_flex_array.skel.h" #include "verifier_btf_unreliable_prog.skel.h" #include "verifier_call_large_imm.skel.h" #include "verifier_cfg.skel.h" @@ -53,6 +54,7 @@ #include "verifier_iterating_callbacks.skel.h" #include "verifier_jeq_infer_not_null.skel.h" #include "verifier_jit_convergence.skel.h" +#include "verifier_kfunc_perfmon.skel.h" #include "verifier_ld_ind.skel.h" #include "verifier_ldsx.skel.h" #include "verifier_leak_ptr.skel.h" @@ -186,6 +188,7 @@ void test_verifier_bpf_get_stack(void) { RUN(verifier_bpf_get_stack); } void test_verifier_bpf_trap(void) { RUN(verifier_bpf_trap); } void test_verifier_bswap(void) { RUN(verifier_bswap); } void test_verifier_btf_ctx_access(void) { RUN(verifier_btf_ctx_access); } +void test_verifier_btf_flex_array(void) { RUN(verifier_btf_flex_array); } void test_verifier_btf_unreliable_prog(void) { RUN(verifier_btf_unreliable_prog); } void test_verifier_call_large_imm(void) { RUN(verifier_call_large_imm); } void test_verifier_cfg(void) { RUN(verifier_cfg); } @@ -216,6 +219,7 @@ void test_verifier_int_ptr(void) { RUN(verifier_int_ptr); } void test_verifier_iterating_callbacks(void) { RUN(verifier_iterating_callbacks); } void test_verifier_jeq_infer_not_null(void) { RUN(verifier_jeq_infer_not_null); } void test_verifier_jit_convergence(void) { RUN(verifier_jit_convergence); } +void test_verifier_kfunc_perfmon(void) { RUN(verifier_kfunc_perfmon); } void test_verifier_load_acquire(void) { RUN(verifier_load_acquire); } void test_verifier_ld_ind(void) { RUN(verifier_ld_ind); } void test_verifier_ldsx(void) { RUN(verifier_ldsx); } diff --git a/tools/testing/selftests/bpf/progs/bpf_ma_ttrace.c b/tools/testing/selftests/bpf/progs/bpf_ma_ttrace.c new file mode 100644 index 000000000000..31b2a962e339 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpf_ma_ttrace.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include "bpf_kfuncs.h" + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(map_flags, BPF_F_NO_PREALLOC); + __uint(max_entries, 4096); + __type(key, int); + __type(value, long); +} htab SEC(".maps"); + +extern const void __per_cpu_offset __ksym; + +int nr_cpus; +int nr_caches; +int in_progress; +int not_freed; + +/* Look at bpf_mem_cache of every cpu that htab allocates its elements from */ +SEC("syscall") +int check_ttrace(void *ctx) +{ + struct bpf_htab *h = bpf_core_cast(&htab, struct bpf_htab); + unsigned long cache = (unsigned long)BPF_CORE_READ(h, ma.cache); + const unsigned long *offsets = &__per_cpu_offset; + struct bpf_mem_cache *c; + unsigned long off; + int cpu; + + nr_caches = 0; + in_progress = 0; + not_freed = 0; + bpf_for(cpu, 0, nr_cpus) { + if (bpf_probe_read_kernel(&off, sizeof(off), offsets + cpu)) + return -1; + c = bpf_core_cast((void *)(cache + off), struct bpf_mem_cache); + if (c->unit_size) + nr_caches++; + if (c->call_rcu_ttrace_in_progress.counter) + in_progress++; + if (c->free_by_rcu_ttrace.first || c->waiting_for_gp_ttrace.first) + not_freed++; + } + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/dynptr_success.c b/tools/testing/selftests/bpf/progs/dynptr_success.c index e0745b6e467e..b668ebd61fc7 100644 --- a/tools/testing/selftests/bpf/progs/dynptr_success.c +++ b/tools/testing/selftests/bpf/progs/dynptr_success.c @@ -10,6 +10,7 @@ #include "errno.h" #define PAGE_SIZE_64K 65536 +#define TEST_SKB_LINEAR_SIZE (sizeof(struct ethhdr) + sizeof(struct iphdr)) char _license[] SEC("license") = "GPL"; @@ -211,6 +212,25 @@ int test_dynptr_skb_data(struct __sk_buff *skb) return 1; } +SEC("?tc") +int test_dynptr_skb_slice_non_linear(struct __sk_buff *skb) +{ + struct bpf_dynptr ptr; + void *data; + + if (bpf_dynptr_from_skb(skb, 0, &ptr)) { + err = 1; + return 1; + } + + /* Ensure we cannot read past the end of the buffer. */ + data = bpf_dynptr_slice(&ptr, TEST_SKB_LINEAR_SIZE + 1, NULL, 1); + if (data) + err = 2; + + return 1; +} + SEC("?tc") int test_dynptr_skb_meta_data(struct __sk_buff *skb) { diff --git a/tools/testing/selftests/bpf/progs/exceptions.c b/tools/testing/selftests/bpf/progs/exceptions.c index c8d716fbd419..91c81971e58c 100644 --- a/tools/testing/selftests/bpf/progs/exceptions.c +++ b/tools/testing/selftests/bpf/progs/exceptions.c @@ -212,6 +212,36 @@ int exception_throw_subprog(struct __sk_buff *ctx) return 0; } +u64 exception_cb_stack_src = 0x1234; + +/* + * The address handed to the helper has to be this callback's own stack + * slot, not one from a frame that is already gone. + */ +__noinline int exception_cb_stack(u64 cookie) +{ + volatile u64 val = 0xdead; + + bpf_probe_read_kernel((void *)&val, sizeof(val), &exception_cb_stack_src); + return val; +} + +/* Throws from a subprogram that has a stack of its own. */ +__noinline static int throwing_subprog_stack(struct __sk_buff *ctx) +{ + volatile u64 pad[4] = {}; + + bpf_throw(pad[0]); + return 0; +} + +SEC("tc") +__exception_cb(exception_cb_stack) +int exception_throw_subprog_stack_cb(struct __sk_buff *ctx) +{ + return throwing_subprog_stack(ctx); +} + __noinline int assert_nz_gfunc(u64 c) { volatile u64 cookie = c; diff --git a/tools/testing/selftests/bpf/progs/iters_state_safety.c b/tools/testing/selftests/bpf/progs/iters_state_safety.c index 646026430e9b..e5bb9fe6d5e5 100644 --- a/tools/testing/selftests/bpf/progs/iters_state_safety.c +++ b/tools/testing/selftests/bpf/progs/iters_state_safety.c @@ -52,6 +52,28 @@ int create_and_destroy(void *ctx) return 0; } +/* fp+0 is not a stack slot. bpf_get_spi(0) used to alias spi 0 (fp-8). */ +SEC("?raw_tp") +__failure __msg("cannot pass in iter at an offset=0") +int destroy_fp0_fail(void *ctx) +{ + struct bpf_iter_num iter; + + asm volatile ("r1 = %[iter];" + "r2 = 0;" + "r3 = 1000;" + "call %[bpf_iter_num_new];" + /* r10 is fp+0, one byte above the top of the BPF stack */ + "r1 = r10;" + "call %[bpf_iter_num_destroy];" + : + : __imm_ptr(iter), ITER_HELPERS + : __clobber_common + ); + + return 0; +} + SEC("?raw_tp") __failure __msg("Unreleased reference id=1") int create_and_forget_to_destroy_fail(void *ctx) diff --git a/tools/testing/selftests/bpf/progs/sock_destroy_prog.c b/tools/testing/selftests/bpf/progs/sock_destroy_prog.c index 9e0bf7a54cec..0a8887543218 100644 --- a/tools/testing/selftests/bpf/progs/sock_destroy_prog.c +++ b/tools/testing/selftests/bpf/progs/sock_destroy_prog.c @@ -7,6 +7,8 @@ #include "bpf_tracing_net.h" __be16 serv_port = 0; +int tw_found = 0; +int tw_destroy_err = 0; int bpf_sock_destroy(struct sock_common *sk) __ksym; @@ -100,6 +102,34 @@ int iter_tcp6_server(struct bpf_iter__tcp *ctx) return 0; } +SEC("iter/tcp") +int iter_tcp6_timewait(struct bpf_iter__tcp *ctx) +{ + struct sock_common *sk_common = ctx->sk_common; + __u64 *val; + int key = 0; + + if (!sk_common) + return 0; + + if (sk_common->skc_family != AF_INET6) + return 0; + + if (!bpf_skc_to_tcp_timewait_sock(sk_common)) + return 0; + + val = bpf_map_lookup_elem(&tcp_conn_sockets, &key); + if (!val) + return 0; + /* The timewait sock inherits the cookie of the closed client sock. */ + if (bpf_get_socket_cookie(sk_common) != *val) + return 0; + + tw_found++; + tw_destroy_err = bpf_sock_destroy(sk_common); + + return 0; +} SEC("iter/udp") int iter_udp6_client(struct bpf_iter__udp *ctx) diff --git a/tools/testing/selftests/bpf/progs/test_spin_lock_fail.c b/tools/testing/selftests/bpf/progs/test_spin_lock_fail.c index f678ee6bd7ea..55282f20fa32 100644 --- a/tools/testing/selftests/bpf/progs/test_spin_lock_fail.c +++ b/tools/testing/selftests/bpf/progs/test_spin_lock_fail.c @@ -14,17 +14,18 @@ struct array_map { __type(key, int); __type(value, struct foo); __uint(max_entries, 1); -} array_map SEC(".maps"); +} array_map SEC(".maps"), array_map_b SEC(".maps"); struct { __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); - __uint(max_entries, 1); + __uint(max_entries, 2); __type(key, int); __type(value, int); __array(values, struct array_map); } map_of_maps SEC(".maps") = { .values = { [0] = &array_map, + [1] = &array_map_b, }, }; @@ -314,4 +315,66 @@ int lock_global_sleepable_subprog_indirect(struct __sk_buff *ctx) return ret; } +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 2); + __type(key, int); + __type(value, struct foo); +} callback_array_map SEC(".maps"); + +struct callback_ctx { + struct foo *value; +}; + +static long lock_different_value(struct bpf_map *map, int *key, + struct foo *value, struct callback_ctx *ctx) +{ + bpf_spin_lock(&value->lock); + bpf_spin_unlock(&ctx->value->lock); + return 0; +} + +static long nest_lock_different_value(struct bpf_map *map, int *key, + struct foo *value, void *data) +{ + struct callback_ctx ctx = { .value = value }; + + bpf_for_each_map_elem(&callback_array_map, lock_different_value, &ctx, 0); + return 0; +} + +SEC("?tc") +int callback_value_lock_identity(void *ctx) +{ + bpf_for_each_map_elem(&callback_array_map, nest_lock_different_value, NULL, 0); + return 0; +} + +static long nest_lock_different_inner_value(struct bpf_map *map, int *key, + struct foo *value, void *data) +{ + struct callback_ctx ctx = { .value = value }; + int inner_key = 1; + void *inner_map; + + inner_map = bpf_map_lookup_elem(&map_of_maps, &inner_key); + if (!inner_map) + return 0; + bpf_for_each_map_elem(inner_map, lock_different_value, &ctx, 0); + return 0; +} + +SEC("?tc") +int callback_inner_map_value_lock_identity(void *ctx) +{ + int inner_key = 0; + void *inner_map; + + inner_map = bpf_map_lookup_elem(&map_of_maps, &inner_key); + if (!inner_map) + return 0; + bpf_for_each_map_elem(inner_map, nest_lock_different_inner_value, NULL, 0); + return 0; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_tc_change_tail_pmtu.c b/tools/testing/selftests/bpf/progs/test_tc_change_tail_pmtu.c new file mode 100644 index 000000000000..5c4c07545bc9 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_tc_change_tail_pmtu.c @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +#define ICMP_SAMPLE_LEN (sizeof(struct iphdr) + 8) +#define ICMP_HDRS_LEN (sizeof(struct iphdr) + sizeof(struct icmphdr)) + +__be16 server_port = 0; +__u16 pmtu = 0; + +long change_tail_ret = 1; +long adjust_room_ret = 0; +bool icmp_sent = false; +bool icmp_err = false; + +static __always_inline __sum16 csum_fold(__wsum csum) +{ + csum = (csum & 0xffff) + (csum >> 16); + csum = (csum & 0xffff) + (csum >> 16); + + return (__sum16)~csum; +} + +SEC("tc/egress") +int change_tail_icmp(struct __sk_buff *skb) +{ + __u8 smac[ETH_ALEN], dmac[ETH_ALEN]; + void *data, *data_end; + struct icmphdr *icmp; + struct ethhdr *eth; + struct tcphdr *tcp; + __be32 saddr, daddr; + struct iphdr *ip; + __wsum csum; + + if (icmp_sent || icmp_err) + return TCX_PASS; + + data = (void *)(long)skb->data; + data_end = (void *)(long)skb->data_end; + + eth = data; + if ((void *)(eth + 1) > data_end) + return TCX_PASS; + if (eth->h_proto != bpf_htons(ETH_P_IP)) + return TCX_PASS; + + ip = (void *)(eth + 1); + if ((void *)(ip + 1) > data_end) + return TCX_PASS; + if (ip->ihl != 5 || ip->protocol != IPPROTO_TCP) + return TCX_PASS; + + tcp = (void *)(ip + 1); + if ((void *)(tcp + 1) > data_end) + return TCX_PASS; + if (tcp->dest != server_port) + return TCX_PASS; + if (bpf_ntohs(ip->tot_len) <= sizeof(*ip) + tcp->doff * 4) + return TCX_PASS; + + __builtin_memcpy(smac, eth->h_source, ETH_ALEN); + __builtin_memcpy(dmac, eth->h_dest, ETH_ALEN); + saddr = ip->saddr; + daddr = ip->daddr; + + change_tail_ret = bpf_skb_change_tail(skb, ETH_HLEN + ICMP_SAMPLE_LEN, 0); + if (change_tail_ret) { + icmp_err = true; + return TCX_PASS; + } + + adjust_room_ret = bpf_skb_adjust_room(skb, ICMP_HDRS_LEN, + BPF_ADJ_ROOM_MAC, + BPF_F_ADJ_ROOM_NO_CSUM_RESET); + if (adjust_room_ret) { + icmp_err = true; + return TCX_DROP; + } + + data = (void *)(long)skb->data; + data_end = (void *)(long)skb->data_end; + + eth = data; + ip = (void *)(eth + 1); + icmp = (void *)(ip + 1); + if ((void *)icmp + sizeof(*icmp) + ICMP_SAMPLE_LEN > data_end) { + icmp_err = true; + return TCX_DROP; + } + + __builtin_memcpy(eth->h_dest, smac, ETH_ALEN); + __builtin_memcpy(eth->h_source, dmac, ETH_ALEN); + + __builtin_memset(icmp, 0, sizeof(*icmp)); + icmp->type = ICMP_DEST_UNREACH; + icmp->code = ICMP_FRAG_NEEDED; + icmp->un.frag.mtu = bpf_htons(pmtu); + + __builtin_memset(ip, 0, sizeof(*ip)); + ip->version = 4; + ip->ihl = 5; + ip->ttl = 64; + ip->protocol = IPPROTO_ICMP; + ip->tot_len = bpf_htons(ICMP_HDRS_LEN + ICMP_SAMPLE_LEN); + ip->saddr = daddr; + ip->daddr = saddr; + + csum = bpf_csum_diff(NULL, 0, (__be32 *)icmp, + sizeof(*icmp) + ICMP_SAMPLE_LEN, 0); + icmp->checksum = csum_fold(csum); + csum = bpf_csum_diff(NULL, 0, (__be32 *)ip, sizeof(*ip), 0); + ip->check = csum_fold(csum); + icmp_sent = true; + return bpf_redirect(skb->ifindex, BPF_F_INGRESS); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/tramp_prog_detach.c b/tools/testing/selftests/bpf/progs/tramp_prog_detach.c new file mode 100644 index 000000000000..507d372167ec --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tramp_prog_detach.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "vmlinux.h" +#include +#include + +char _license[] SEC("license") = "GPL"; + +int pid; +void *fault_addr; +__u64 ts; + +static int do_sleepable(void) +{ + char dst; + + if (bpf_get_current_pid_tgid() >> 32 != pid) + return 0; + + ts = bpf_ktime_get_ns(); + /* blocks for as long as user space wants when fault_addr is armed */ + bpf_copy_from_user(&dst, sizeof(dst), fault_addr); + return 0; +} + +static int do_victim(void) +{ + if (bpf_get_current_pid_tgid() >> 32 != pid) + return 0; + + ts = bpf_ktime_get_ns(); + return 0; +} + +SEC("?fentry.s/bpf_fentry_test1") +int BPF_PROG(fentry_sleepable, int a) +{ + return do_sleepable(); +} + +SEC("?fentry/bpf_fentry_test1") +int BPF_PROG(fentry_victim, int a) +{ + return do_victim(); +} + +SEC("?fexit.s/bpf_fentry_test1") +int BPF_PROG(fexit_sleepable, int a, int ret) +{ + return do_sleepable(); +} + +SEC("?fexit/bpf_fentry_test1") +int BPF_PROG(fexit_victim, int a, int ret) +{ + return do_victim(); +} diff --git a/tools/testing/selftests/bpf/progs/verifier_align.c b/tools/testing/selftests/bpf/progs/verifier_align.c index 3e52686515ca..0083ef8b8ba7 100644 --- a/tools/testing/selftests/bpf/progs/verifier_align.c +++ b/tools/testing/selftests/bpf/progs/verifier_align.c @@ -284,7 +284,7 @@ __msg("26: {{.*}} R5=pkt(r=8,imm=14)") */ __msg("28: {{.*}} R4={{[^)]*}}var_off=(0x2; 0x7fc){{.*}} R5={{[^)]*}}var_off=(0x2; 0x7fc)") /* Constant is added to R5 again, setting reg->off to 18. */ -__msg("29: {{.*}} R5=pkt(id=3,{{[^)]*}}var_off=(0x2; 0x7fc)") +__msg("29: {{.*}} R5=pkt(id=3,{{[^)]*}}var_off=(0x2; 0xffc)") /* And once more we add a variable; resulting {{[^)]*}}var_off * is still (4n), fixed offset is not changed. * Also, we create a new reg->id. @@ -359,7 +359,7 @@ __msg("7: {{.*}} R6={{[^)]*}}var_off=(0x0; 0x3fc)") __msg("8: {{.*}} R6={{[^)]*}}var_off=(0x2; 0x7fc)") /* Packet pointer has (4n+2) offset */ __msg("11: {{.*}} R5={{[^)]*}}var_off=(0x2; 0x7fc)") -__msg("12: {{.*}} R4={{[^)]*}}var_off=(0x2; 0x7fc)") +__msg("12: {{.*}} R4={{[^)]*}}var_off=(0x2; 0xffc)") /* At the time the word size load is performed from R5, * its total fixed offset is NET_IP_ALIGN + reg->off (0) * which is 2. Then the variable offset is (4n+2), so @@ -375,7 +375,7 @@ __msg("17: {{.*}} R6={{[^)]*}}var_off=(0x0; 0x3fc)") * another (4n+2). */ __msg("19: {{.*}} R5={{[^)]*}}var_off=(0x2; 0xffc)") -__msg("20: {{.*}} R4={{[^)]*}}var_off=(0x2; 0xffc)") +__msg("20: {{.*}} R4={{[^)]*}}var_off=(0x2; 0x1ffc)") /* At the time the word size load is performed from R5, * its total fixed offset is NET_IP_ALIGN + reg->off (0) * which is 2. Then the variable offset is (4n+2), so diff --git a/tools/testing/selftests/bpf/progs/verifier_arena.c b/tools/testing/selftests/bpf/progs/verifier_arena.c index 815f342eb4b0..3e33766547c0 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena.c @@ -561,6 +561,55 @@ int arena_ptr_add_arena_ptr(void *ctx) return 0; } +SEC("syscall") +__failure __msg("same insn cannot be used with and without arena pointer") +int mixed_arena_scalar_alu64_scalar_first(void *ctx) +{ + volatile register __u64 reg asm("r3"); + __u32 pick_arena = bpf_get_prandom_u32(); + + reg = 1ULL << 32; + + if (pick_arena) { + asm volatile ( + "r9 = %[arena] ll;" + "%[reg] = 0;" + "%[reg] = addr_space_cast(%[reg], 0x0, 0x1);" + : [reg] "=r"(reg) + : __imm_addr(arena) + : "r9" + ); + } + + reg += 1; + + return 0; +} + +SEC("syscall") +__failure __msg("same insn cannot be used with and without arena pointer") +int mixed_arena_scalar_alu64_arena_first(void *ctx) +{ + volatile register __u64 reg asm("r3"); + __u32 pick_scalar = bpf_get_prandom_u32(); + + asm volatile ( + "r9 = %[arena] ll;" + "%[reg] = 0;" + "%[reg] = addr_space_cast(%[reg], 0x0, 0x1);" + : [reg] "=r"(reg) + : __imm_addr(arena) + : "r9" + ); + + if (pick_scalar) + reg = 1ULL << 32; + + reg += 1; + + return 0; +} + SEC("syscall") __success __retval(0) int scalar_xor_arena_ptr(void *ctx) diff --git a/tools/testing/selftests/bpf/progs/verifier_async_cb_context.c b/tools/testing/selftests/bpf/progs/verifier_async_cb_context.c index e0926767bbd3..1b653bfb63eb 100644 --- a/tools/testing/selftests/bpf/progs/verifier_async_cb_context.c +++ b/tools/testing/selftests/bpf/progs/verifier_async_cb_context.c @@ -9,6 +9,11 @@ char _license[] SEC("license") = "GPL"; +struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; +void bpf_task_release(struct task_struct *p) __ksym; +void bpf_rcu_read_lock(void) __ksym; +void bpf_rcu_read_unlock(void) __ksym; + /* Timer tests */ struct timer_elem { @@ -164,6 +169,7 @@ int syscall_btf_find_prog(void *ctx) struct wq_elem { struct bpf_wq w; + struct task_struct __kptr *task; }; struct { @@ -217,6 +223,106 @@ int wq_sleepable_prog(void *ctx) return 0; } +__noinline int wq_global_acquire(void) +{ + struct task_struct *task, *acquired; + struct wq_elem *val; + int key = 0; + + val = bpf_map_lookup_elem(&wq_map, &key); + if (!val) + return 0; + + task = val->task; + if (!task) + return 0; + + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + return 0; +} + +static int wq_global_rcu_cb(void *map, int *key, void *value) +{ + wq_global_acquire(); + return 0; +} + +SEC("fentry/bpf_fentry_test1") +__failure __msg("R1 must be a rcu pointer") +int wq_global_rcu_prog(void *ctx) +{ + struct wq_elem *val; + int key = 0; + + val = bpf_map_lookup_elem(&wq_map, &key); + if (!val) + return 0; + + bpf_wq_init(&val->w, &wq_map, 0); + bpf_wq_set_callback(&val->w, wq_global_rcu_cb, 0); + return 0; +} + +static int wq_global_rcu_lock_cb(void *map, int *key, void *value) +{ + bpf_rcu_read_lock(); + wq_global_acquire(); + bpf_rcu_read_unlock(); + return 0; +} + +SEC("fentry/bpf_fentry_test1") +__success +int wq_global_rcu_lock_prog(void *ctx) +{ + struct wq_elem *val; + int key = 0; + + /* Verify the same global subprog in non-sleepable and protected contexts. */ + wq_global_acquire(); + + val = bpf_map_lookup_elem(&wq_map, &key); + if (!val) + return 0; + + bpf_wq_init(&val->w, &wq_map, 0); + bpf_wq_set_callback(&val->w, wq_global_rcu_lock_cb, 0); + return 0; +} + +__weak __noinline int wq_global_no_rcu(void) +{ + return 0; +} + +static int wq_global_no_rcu_cb(void *map, int *key, void *value) +{ + wq_global_no_rcu(); + return 0; +} + +SEC("fentry/bpf_fentry_test1") +__success __log_level(4) +__msg("subprog {{[0-9]+}} (wq_global_no_rcu) global insns_self 4 insns_total 4 stack 0") +int wq_global_no_rcu_prog(void *ctx) +{ + struct wq_elem *val; + int key = 0; + + /* Verify the same global in non-sleepable and unprotected contexts. */ + wq_global_no_rcu(); + + val = bpf_map_lookup_elem(&wq_map, &key); + if (!val) + return 0; + + bpf_wq_init(&val->w, &wq_map, 0); + bpf_wq_set_callback(&val->w, wq_global_no_rcu_cb, 0); + return 0; +} + /* Task work tests */ struct task_work_elem { diff --git a/tools/testing/selftests/bpf/progs/verifier_btf_flex_array.c b/tools/testing/selftests/bpf/progs/verifier_btf_flex_array.c new file mode 100644 index 000000000000..59b84261f622 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_btf_flex_array.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#include "bpf_experimental.h" +#include "bpf_misc.h" + +struct test_empty_event {}; + +struct test_flex_batch { + int nr; + struct test_empty_event events[]; +}; + +struct map_value { + struct test_flex_batch __kptr *batch; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, int); + __type(value, struct map_value); + __uint(max_entries, 1); +} batches SEC(".maps"); + +SEC("syscall") +__description("btf walk into flexible array of zero-sized elements") +__failure __msg("access beyond struct test_flex_batch at off 4 size 1") +int stash_and_peek(void *ctx) +{ + struct test_flex_batch *b, *old; + struct map_value *v; + int key = 0; + + v = bpf_map_lookup_elem(&batches, &key); + if (!v) + return 0; + + b = bpf_obj_new(struct test_flex_batch); + if (!b) + return 0; + b->nr = 1; + + old = bpf_kptr_xchg(&v->batch, b); + if (old) + bpf_obj_drop(old); + + b = v->batch; + if (!b) + return 0; + + return b->nr + *(char *)&b->events[0]; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c b/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c index 915a9707298b..139ff019d87d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c +++ b/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c @@ -920,4 +920,182 @@ l1_%=: r0 = *(u8*)(r9 + 0); \ : __clobber_all); } +SEC("tc") +__description("direct packet access: 8-aligned offset, check p + 8, load 8 bytes at p") +__success __retval(0) __flag(BPF_F_ANY_ALIGNMENT) +__naked void pkt_same_id_check_copy_load_base(void) +{ + asm volatile (" \ + r2 = *(u32*)(r1 + %[__sk_buff_data]); \ + r3 = *(u32*)(r1 + %[__sk_buff_data_end]); \ + r4 = *(u32*)(r1 + %[__sk_buff_mark]); \ + r4 &= 0x38; \ + if r4 > 50 goto l0_%=; \ + /* r4 is a multiple of 8, at most 48 */ \ + r5 = r2; \ + r5 += r4; \ + r6 = r5; \ + r6 += 8; \ + if r6 > r3 goto l0_%=; \ + /* [r5, r5 + 8) is in the packet */ \ + r0 = *(u64*)(r5 + 0); \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), + __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)), + __imm_const(__sk_buff_mark, offsetof(struct __sk_buff, mark)) + : __clobber_all); +} + +SEC("tc") +__description("direct packet access: 8-aligned offset, check p, load past it via p + 8") +__failure __msg("invalid access to packet, off=51 size=1") +__naked void pkt_same_id_check_base_load_via_copy(void) +{ + asm volatile (" \ + r2 = *(u32*)(r1 + %[__sk_buff_data]); \ + r3 = *(u32*)(r1 + %[__sk_buff_data_end]); \ + r4 = *(u32*)(r1 + %[__sk_buff_mark]); \ + r4 &= 0x38; \ + if r4 > 50 goto l0_%=; \ + /* r4 is a multiple of 8, at most 48 */ \ + r5 = r2; \ + r5 += r4; \ + r6 = r5; \ + r6 += 8; \ + if r5 > r3 goto l0_%=; \ + /* r6 - 7 is r5 + 1 */ \ + r0 = *(u8*)(r6 - 7); \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), + __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)), + __imm_const(__sk_buff_mark, offsetof(struct __sk_buff, mark)) + : __clobber_all); +} + +SEC("tc") +__description("direct packet access: 8-aligned offset, check p + 4, 4-byte load at p + 2") +__failure __msg("invalid access to packet, off=52 size=4") +__naked void pkt_same_id_check_base_4_load_via_copy(void) +{ + asm volatile (" \ + r2 = *(u32*)(r1 + %[__sk_buff_data]); \ + r3 = *(u32*)(r1 + %[__sk_buff_data_end]); \ + r4 = *(u32*)(r1 + %[__sk_buff_mark]); \ + r4 &= 0x38; \ + if r4 > 50 goto l0_%=; \ + /* r4 is a multiple of 8, at most 48 */ \ + r5 = r2; \ + r5 += r4; \ + r6 = r5; \ + r6 += 8; \ + r7 = r5; \ + r7 += 4; \ + if r7 > r3 goto l0_%=; \ + /* [r5, r5 + 4) is in the packet, [r5 + 2, r5 + 6) may not be */ \ + r0 = *(u32*)(r6 - 6); \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), + __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)), + __imm_const(__sk_buff_mark, offsetof(struct __sk_buff, mark)) + : __clobber_all); +} + +SEC("tc") +__description("direct packet access: checked pointer minus non-negative unknown keeps range") +__success __retval(0) __flag(BPF_F_ANY_ALIGNMENT) +__naked void pkt_sub_unknown_keeps_range(void) +{ + asm volatile (" \ + r2 = *(u32*)(r1 + %[__sk_buff_data]); \ + r3 = *(u32*)(r1 + %[__sk_buff_data_end]); \ + r4 = *(u32*)(r1 + %[__sk_buff_mark]); \ + r4 &= 0x1f; \ + r4 += 8; \ + r5 = r2; \ + r5 += 40; \ + if r5 > r3 goto l0_%=; \ + /* r5 is 8 to 39 bytes below the checked pointer */ \ + r5 -= r4; \ + r0 = *(u64*)(r5 + 0); \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), + __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)), + __imm_const(__sk_buff_mark, offsetof(struct __sk_buff, mark)) + : __clobber_all); +} + +SEC("tc") +__description("direct packet access: no pruning of a path whose checks prove fewer bytes") +__failure __msg("invalid access to packet, off=255 size=8") +__flag(BPF_F_ANY_ALIGNMENT) __flag(BPF_F_TEST_STATE_FREQ) +__naked void pkt_same_id_pruning(void) +{ + asm volatile (" \ + r2 = *(u32*)(r1 + %[__sk_buff_data]); \ + r3 = *(u32*)(r1 + %[__sk_buff_data_end]); \ + r4 = *(u32*)(r1 + %[__sk_buff_mark]); \ + r0 = *(u32*)(r1 + %[__sk_buff_priority]); \ + r4 &= 0xff; \ + r5 = r2; \ + r5 += r4; \ + if r0 != 0 goto l1_%=; \ + /* this path proves [r5, r5 + 8) */ \ + r6 = r5; \ + r6 += 8; \ + if r6 > r3 goto l0_%=; \ + goto l2_%=; \ +l1_%=: /* this path proves [r5, r5 + 7) */ \ + if r5 > r3 goto l0_%=; \ + r6 = r5; \ + r6 += 7; \ + if r6 > r3 goto l0_%=; \ +l2_%=: r0 = *(u64*)(r5 + 0); \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), + __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)), + __imm_const(__sk_buff_mark, offsetof(struct __sk_buff, mark)), + __imm_const(__sk_buff_priority, offsetof(struct __sk_buff, priority)) + : __clobber_all); +} + +SEC("tc") +__description("direct packet access: spilled copy of checked pointer, load past range") +__failure __msg("invalid access to packet, off=262 size=1, R7(id={{[0-9]+}},off=262,r=262)") +__naked void pkt_spilled_copy_gets_range(void) +{ + asm volatile (" \ + r2 = *(u32*)(r1 + %[__sk_buff_data]); \ + r3 = *(u32*)(r1 + %[__sk_buff_data_end]); \ + r4 = *(u32*)(r1 + %[__sk_buff_mark]); \ + r4 &= 0xff; \ + r5 = r2; \ + r5 += r4; \ + *(u64*)(r10 - 8) = r5; \ + /* proves only bytes before r5 */ \ + if r5 > r3 goto l0_%=; \ + r6 = r5; \ + r6 += 7; \ + if r6 > r3 goto l0_%=; \ + /* [r5, r5 + 7) is in the packet */ \ + r7 = *(u64*)(r10 - 8); \ + r0 = *(u8*)(r7 + 7); \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), + __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)), + __imm_const(__sk_buff_mark, offsetof(struct __sk_buff, mark)) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c b/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c index a3d2af8dc839..dcc2dd46751a 100644 --- a/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c +++ b/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c @@ -350,4 +350,57 @@ int anything_to_untrusted_mem(void *ctx) return 0; } +struct pkt_arg { + __u64 x; + __u8 pad[56]; +}; + +__weak int subprog_pkt_ptr_no_change(struct pkt_arg *p) +{ + if (!p) + return 0; + + return p->x; +} + +SEC("?tc") +__success +int pkt_ptr_to_global_mem_arg_no_change(struct __sk_buff *skb) +{ + void *data = (void *)(long)skb->data; + void *data_end = (void *)(long)skb->data_end; + struct pkt_arg *p = data; + + if ((void *)(p + 1) > data_end) + return 0; + + return subprog_pkt_ptr_no_change(p); +} + +__weak int subprog_pkt_ptr_changes_data(struct __sk_buff *skb __arg_ctx, + struct pkt_arg *p) +{ + if (!p) + return 0; + + bpf_skb_pull_data(skb, 0); + return p->x; +} + +SEC("?tc") +__failure __log_level(2) +__msg("R2 is a packet pointer, but func#{{[0-9]+}} may change packet data") +__msg("Caller passes invalid args into func#{{[0-9]+}} ('subprog_pkt_ptr_changes_data')") +int pkt_ptr_to_global_mem_arg_changes_data(struct __sk_buff *skb) +{ + void *data = (void *)(long)skb->data; + void *data_end = (void *)(long)skb->data_end; + struct pkt_arg *p = data; + + if ((void *)(p + 1) > data_end) + return 0; + + return subprog_pkt_ptr_changes_data(skb, p); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_gotox.c b/tools/testing/selftests/bpf/progs/verifier_gotox.c index 5b18c9a27717..0e27c2c79c57 100644 --- a/tools/testing/selftests/bpf/progs/verifier_gotox.c +++ b/tools/testing/selftests/bpf/progs/verifier_gotox.c @@ -47,6 +47,54 @@ DEFINE_SIMPLE_JUMP_TABLE_PROG(reserved_field_src_reg, BPF_REG_1, 0, 0, __fa DEFINE_SIMPLE_JUMP_TABLE_PROG(reserved_field_non_zero_off, BPF_REG_0, 1, 0, __failure __msg("BPF_JA|BPF_X uses reserved fields")) DEFINE_SIMPLE_JUMP_TABLE_PROG(reserved_field_non_zero_imm, BPF_REG_0, 0, 1, __failure __msg("BPF_JA|BPF_X uses reserved fields")) +#define DEFINE_TERMINAL_GOTOX_PROG(NAME, BASE) \ + __naked void NAME(void) \ + { \ + asm volatile (" \ + .pushsection .jumptables,\"\",@progbits; \ +jt0_%=: \ + .quad ret0_%= - " BASE "; \ + .size jt0_%=, 8; \ + .global jt0_%=; \ + .popsection; \ + \ + r0 = jt0_%= ll; \ + r0 = *(u64 *)(r0 + 0); \ + goto end_%=; \ +ret0_%=: \ + r0 = 0; \ + exit; \ +end_%=: \ + .8byte %[gotox_r0]; \ +" : \ + : __imm_insn(gotox_r0, BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, \ + BPF_REG_0, 0, 0, 0)) \ + : __clobber_all); \ + } + +SEC("socket") +__success __retval(0) +DEFINE_TERMINAL_GOTOX_PROG(jump_table_terminal_gotox, "socket") + +static __noinline __used +DEFINE_TERMINAL_GOTOX_PROG(terminal_gotox_subprog1, ".text") + +static __noinline __used int terminal_gotox_subprog2(void) +{ + return 0; +} + +SEC("socket") +__success __retval(0) +__naked void jump_table_terminal_gotox_subprog(void) +{ + asm volatile (" \ + call terminal_gotox_subprog1; \ + call terminal_gotox_subprog2; \ + exit; \ +" ::: __clobber_all); +} + /* * Gotox is forbidden when there is no jump table loaded * which points to the sub-function where the gotox is used diff --git a/tools/testing/selftests/bpf/progs/verifier_kfunc_perfmon.c b/tools/testing/selftests/bpf/progs/verifier_kfunc_perfmon.c new file mode 100644 index 000000000000..76c39ef30e96 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_kfunc_perfmon.c @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include "bpf_misc.h" + +void *user_ptr; +char dynptr_buf[8]; +u64 kaddr; + +extern struct kmem_cache *bpf_get_kmem_cache(u64 addr) __ksym; + +SEC("socket") +__success +__caps_unpriv(CAP_BPF) +__failure_unpriv +__msg_unpriv("bpf_rdonly_cast is allowed only to CAP_PERFMON and CAP_SYS_ADMIN") +int rdonly_cast_noperfmon(void *ctx) +{ + char *p = bpf_rdonly_cast(0, 0); + + return p[0x7fff]; +} + +SEC("socket") +__success +__caps_unpriv(CAP_BPF) +__failure_unpriv +__msg_unpriv("bpf_probe_read_kernel_dynptr is allowed only to CAP_PERFMON and CAP_SYS_ADMIN") +int probe_read_kernel_dynptr_noperfmon(void *ctx) +{ + struct bpf_dynptr dptr; + + bpf_dynptr_from_mem(dynptr_buf, sizeof(dynptr_buf), 0, &dptr); + bpf_probe_read_kernel_dynptr(&dptr, 0, sizeof(dynptr_buf), user_ptr); + return 0; +} + +SEC("socket") +__success +__caps_unpriv(CAP_BPF) +__failure_unpriv +__msg_unpriv("bpf_stream_vprintk is allowed only to CAP_PERFMON and CAP_SYS_ADMIN") +int stream_vprintk_noperfmon(void *ctx) +{ + bpf_stream_printk(BPF_STDOUT, "%pB", (void *)kaddr); + return 0; +} + +SEC("socket") +__success +__caps_unpriv(CAP_BPF) +__failure_unpriv +__msg_unpriv("bpf_get_kmem_cache is allowed only to CAP_PERFMON and CAP_SYS_ADMIN") +int get_kmem_cache_noperfmon(void *ctx) +{ + return !!bpf_get_kmem_cache(kaddr); +} + +__weak int subprog_untrusted_read(void *p __arg_untrusted) +{ + return *(char *)p; +} + +SEC("socket") +__success +__caps_unpriv(CAP_BPF) +__failure_unpriv +__msg_unpriv("rdonly_untrusted_mem access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN") +int arg_untrusted_read_noperfmon(void *ctx) +{ + return subprog_untrusted_read(0); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_loops1.c b/tools/testing/selftests/bpf/progs/verifier_loops1.c index d248ce877f14..48a966cda199 100644 --- a/tools/testing/selftests/bpf/progs/verifier_loops1.c +++ b/tools/testing/selftests/bpf/progs/verifier_loops1.c @@ -303,4 +303,40 @@ __naked void maybe_exit_scc_bug1(void) ::: __clobber_all); } +/* + * The loop reads zero from the caller's stack on its first iteration and + * one from the callee's stack on its second iteration. At the loop header, + * only the frame number of the pointer in r1 changes. + */ +static __naked __noinline __used +void loop_stack_frames_reg(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 1;" +"1:" + "r0 = *(u64 *)(r1 + 0);" + "if r0 != 0 goto 2f;" + "r1 = r10;" + "r1 += -8;" + "goto 1b;" +"2:" + "exit;" + ::: __clobber_all); +} + +SEC("xdp") +__description("bounded loop changing stack frame in a register") +__success __retval(1) +__flag(BPF_F_TEST_STATE_FREQ) +__naked void bounded_loop_stack_frames_reg(void) +{ + asm volatile ( + "*(u64 *)(r10 - 8) = 0;" + "r1 = r10;" + "r1 += -8;" + "call loop_stack_frames_reg;" + "exit;" + ::: __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_meta_access.c b/tools/testing/selftests/bpf/progs/verifier_meta_access.c index 62235f032ffe..c87e0be4b2ff 100644 --- a/tools/testing/selftests/bpf/progs/verifier_meta_access.c +++ b/tools/testing/selftests/bpf/progs/verifier_meta_access.c @@ -281,4 +281,34 @@ l0_%=: r0 = 0; \ : __clobber_all); } +SEC("xdp") +__description("meta access, 8-aligned offset, check p, load a byte at p + 1 via p + 8") +__failure __msg("invalid access to packet, off=51 size=1") +__naked void meta_access_check_base_load_via_copy(void) +{ + asm volatile (" \ + r9 = r1; \ + call %[bpf_get_prandom_u32]; \ + r4 = r0; \ + r4 &= 0x38; \ + if r4 > 50 goto l0_%=; \ + /* r4 is a multiple of 8, at most 48 */ \ + r2 = *(u32*)(r9 + %[xdp_md_data_meta]); \ + r3 = *(u32*)(r9 + %[xdp_md_data]); \ + r5 = r2; \ + r5 += r4; \ + r6 = r5; \ + r6 += 8; \ + if r5 > r3 goto l0_%=; \ + /* r6 - 7 is r5 + 1 */ \ + r0 = *(u8*)(r6 - 7); \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32), + __imm_const(xdp_md_data, offsetof(struct xdp_md, data)), + __imm_const(xdp_md_data_meta, offsetof(struct xdp_md, data_meta)) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_sock.c b/tools/testing/selftests/bpf/progs/verifier_sock.c index 4f2f3209eec8..bf9f6fb6582c 100644 --- a/tools/testing/selftests/bpf/progs/verifier_sock.c +++ b/tools/testing/selftests/bpf/progs/verifier_sock.c @@ -88,6 +88,44 @@ l0_%=: r0 = *(u32*)(r1 + %[bpf_sock_family]); \ : __clobber_all); } +SEC("socket") +__description("skb->sk: sk->rx_queue_mapping [no sign extension]") +__success __success_unpriv __retval(0) +__naked void sk_rx_queue_mapping_no_sign_ext(void) +{ + asm volatile (" \ + r1 = *(u64*)(r1 + %[__sk_buff_sk]); \ + if r1 != 0 goto l0_%=; \ + r0 = 0xdead; \ + exit; \ +l0_%=: r0 = *(u32*)(r1 + %[bpf_sock_rx_queue_mapping]); \ + r0 >>= 32; \ + exit; \ +" : + : __imm_const(__sk_buff_sk, offsetof(struct __sk_buff, sk)), + __imm_const(bpf_sock_rx_queue_mapping, offsetof(struct bpf_sock, rx_queue_mapping)) + : __clobber_all); +} + +SEC("socket") +__description("skb->sk: sk->rx_queue_mapping [narrow load mask]") +__success __success_unpriv __retval(0) +__naked void sk_rx_queue_mapping_narrow_load_mask(void) +{ + asm volatile (" \ + r1 = *(u64*)(r1 + %[__sk_buff_sk]); \ + if r1 != 0 goto l0_%=; \ + r0 = 0xdead; \ + exit; \ +l0_%=: r0 = *(u16*)(r1 + %[bpf_sock_rx_queue_mapping]); \ + r0 >>= 16; \ + exit; \ +" : + : __imm_const(__sk_buff_sk, offsetof(struct __sk_buff, sk)), + __imm_const(bpf_sock_rx_queue_mapping, offsetof(struct bpf_sock, rx_queue_mapping)) + : __clobber_all); +} + SEC("cgroup/skb") __description("skb->sk: sk->type [fullsock field]") __failure __msg("invalid sock_common access") diff --git a/tools/testing/selftests/bpf/progs/verifier_xdp_direct_packet_access.c b/tools/testing/selftests/bpf/progs/verifier_xdp_direct_packet_access.c index 0b86d95a4133..9866bc154194 100644 --- a/tools/testing/selftests/bpf/progs/verifier_xdp_direct_packet_access.c +++ b/tools/testing/selftests/bpf/progs/verifier_xdp_direct_packet_access.c @@ -1719,4 +1719,39 @@ l0_%=: r0 = 0; \ : __clobber_all); } +SEC("xdp") +__description("XDP pkt regsafe preserves packet pointer class displacement") +__failure __msg("R2 min value is outside of the allowed memory range") +__flag(BPF_F_ANY_ALIGNMENT) __flag(BPF_F_TEST_STATE_FREQ) +__naked void pkt_regsafe_class_displacement(void) +{ + asm volatile (" \ + r8 = *(u32 *)(r1 + %[xdp_md_data_end]); \ + r9 = *(u32 *)(r1 + %[xdp_md_data]); \ + r4 = *(u32 *)(r1 + %[xdp_md_rx_queue_index]); \ + r4 &= 15; \ + r0 = *(u32 *)(r1 + %[xdp_md_ingress_ifindex]); \ + if r0 != 0 goto l0_%=; \ + r2 = r9; \ + r2 += r4; \ + r3 = r2; \ + r3 += 8; \ + goto l1_%=; \ +l0_%=: r4 &= 3; \ + r4 += 8; \ + r2 = r9; \ + r2 += r4; \ + r3 = r2; \ +l1_%=: if r3 > r8 goto l2_%=; \ + r0 = *(u64 *)(r2 + 0); \ +l2_%=: r0 = 0; \ + exit; \ +" : + : __imm_const(xdp_md_data, offsetof(struct xdp_md, data)), + __imm_const(xdp_md_data_end, offsetof(struct xdp_md, data_end)), + __imm_const(xdp_md_rx_queue_index, offsetof(struct xdp_md, rx_queue_index)), + __imm_const(xdp_md_ingress_ifindex, offsetof(struct xdp_md, ingress_ifindex)) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/testing_helpers.c b/tools/testing/selftests/bpf/testing_helpers.c index c970e7793dfc..9f672d0c9dda 100644 --- a/tools/testing/selftests/bpf/testing_helpers.c +++ b/tools/testing/selftests/bpf/testing_helpers.c @@ -13,6 +13,7 @@ #include "test_progs.h" #include "testing_helpers.h" #include +#include int parse_num_list(const char *s, bool **num_set, int *num_set_len) { @@ -459,6 +460,33 @@ int kern_sync_rcu(void) return syscall(__NR_membarrier, MEMBARRIER_CMD_SHARED, 0, 0); } +int uffd_block_page(void *fault_addr) +{ + struct uffdio_register uffd_register = {}; + struct uffdio_api uffd_api = {}; + int uffd; + + uffd = syscall(__NR_userfaultfd, O_CLOEXEC); + if (uffd < 0) + return -errno; + + uffd_api.api = UFFD_API; + uffd_api.features = 0; + if (ioctl(uffd, UFFDIO_API, &uffd_api)) { + close(uffd); + return -1; + } + + uffd_register.range.start = (unsigned long)fault_addr; + uffd_register.range.len = getpagesize(); + uffd_register.mode = UFFDIO_REGISTER_MODE_MISSING; + if (ioctl(uffd, UFFDIO_REGISTER, &uffd_register)) { + close(uffd); + return -1; + } + return uffd; +} + int get_xlated_program(int fd_prog, struct bpf_insn **buf, __u32 *cnt) { __u32 buf_element_size = sizeof(struct bpf_insn); diff --git a/tools/testing/selftests/bpf/testing_helpers.h b/tools/testing/selftests/bpf/testing_helpers.h index 2edc6fb7fc52..1f83b6080716 100644 --- a/tools/testing/selftests/bpf/testing_helpers.h +++ b/tools/testing/selftests/bpf/testing_helpers.h @@ -36,6 +36,8 @@ __u64 read_perf_max_sample_freq(void); int load_bpf_testmod(bool verbose); int unload_bpf_testmod(bool verbose); int kern_sync_rcu(void); +/* returns a userfaultfd that makes accesses to fault_addr's page block */ +int uffd_block_page(void *fault_addr); int finit_module(int fd, const char *param_values, int flags); int delete_module(const char *name, int flags); int load_module(const char *path, bool verbose); diff --git a/tools/testing/selftests/cgroup/test_cpuset_prs.sh b/tools/testing/selftests/cgroup/test_cpuset_prs.sh index 131d8b4551ef..7efd5e645767 100755 --- a/tools/testing/selftests/cgroup/test_cpuset_prs.sh +++ b/tools/testing/selftests/cgroup/test_cpuset_prs.sh @@ -298,6 +298,8 @@ TEST_MATRIX=( " C0-4:X2-4 C1-4:X2-4:P2 C2-4:X4:P1 \ . . . X1 . 0 A1:0-1|A2:2-4|A3:2-4 \ A1:P0|A2:P2|A3:P-1 2-4" + " CX1-3:P1 CX1-3 CX1-3 . . . P1 . 0 A1:1-3|A2:1-3|A3:1-3 \ + A1:P1|A2:P0|A3:P-1" # Remote partition offline tests " C0-3 C1-3 C2-3 . X2-3 X2-3 X2-3:P2:O2=0 . 0 A1:0-1|A2:1|A3:3 A1:P0|A3:P2 2-3" diff --git a/tools/testing/selftests/cgroup/test_memcontrol.c b/tools/testing/selftests/cgroup/test_memcontrol.c index 3a84d068fbf3..0ed82347044e 100644 --- a/tools/testing/selftests/cgroup/test_memcontrol.c +++ b/tools/testing/selftests/cgroup/test_memcontrol.c @@ -30,7 +30,7 @@ static int page_size; int get_temp_fd(void) { - return open(".", O_TMPFILE | O_RDWR | O_EXCL); + return open(".", O_TMPFILE | O_RDWR | O_EXCL, 0600); } int alloc_pagecache(int fd, size_t size) diff --git a/tools/testing/selftests/cgroup/test_zswap.c b/tools/testing/selftests/cgroup/test_zswap.c index 609c48f38524..8df54b59513a 100644 --- a/tools/testing/selftests/cgroup/test_zswap.c +++ b/tools/testing/selftests/cgroup/test_zswap.c @@ -20,6 +20,7 @@ static int page_size; #define PATH_ZSWAP "/sys/module/zswap" #define PATH_ZSWAP_ENABLED "/sys/module/zswap/parameters/enabled" +#define PATH_ZSWAP_SHRINKER_ENABLED "/sys/module/zswap/parameters/shrinker_enabled" #define PATH_ZSWAP_STORED_PAGES "/sys/kernel/debug/zswap/stored_pages" static int read_int(const char *path, size_t *value) @@ -444,6 +445,16 @@ static int test_zswap_writeback_disabled(const char *root) return test_zswap_writeback(root, false); } +static bool zswap_shrinker_enabled(void) +{ + char value[2]; + + if (read_text(PATH_ZSWAP_SHRINKER_ENABLED, value, sizeof(value)) <= 0) + return 0; + + return value[0] == 'Y'; +} + /* * When trying to store a memcg page in zswap, if the memcg hits its memory * limit in zswap, writeback should affect only the zswapped pages of that @@ -453,6 +464,7 @@ static int test_no_invasive_cgroup_shrink(const char *root) { int ret = KSFT_FAIL; unsigned int off; + long zswpwb_before, zswpwb_after, zswpwb_target; size_t allocation_size = page_size * 1024; unsigned int nr_pages = allocation_size / page_size; char zswap_max_buf[32], mem_max_buf[32]; @@ -488,6 +500,14 @@ static int test_no_invasive_cgroup_shrink(const char *root) if (cg_read_key_long(zw_group, "memory.stat", "zswapped") < 1) goto out; + /* If the shrinker is enabled, try to let the writebacks finish first */ + if (zswap_shrinker_enabled()) + sleep(5); + + zswpwb_before = get_cg_wb_count(zw_group); + if (zswpwb_before < 0) + goto out; + /* Push wb_group memory into zswap with hard-to-compress data to trigger wb */ if (cg_enter_current(wb_group)) goto out; @@ -500,9 +520,13 @@ static int test_no_invasive_cgroup_shrink(const char *root) getrandom(&wb_allocation[off], page_size/4, 0); } - /* Verify that only zswapped memory from gwb_group has been written back */ - if (wait_for_writeback(wb_group, 5000) > 0 && get_cg_wb_count(zw_group) == 0) + /* Verify that only zswapped memory from wb_group has been written back */ + zswpwb_target = wait_for_writeback(wb_group, 5000); + zswpwb_after = get_cg_wb_count(zw_group); + + if (zswpwb_target > 0 && zswpwb_before == zswpwb_after) ret = KSFT_PASS; + out: cg_enter_current(root); if (zw_group) { diff --git a/tools/testing/selftests/filesystems/mntns_cleanup/.gitignore b/tools/testing/selftests/filesystems/mntns_cleanup/.gitignore deleted file mode 100644 index 493fbcf8d9ec..000000000000 --- a/tools/testing/selftests/filesystems/mntns_cleanup/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -mntns_cleanup_test diff --git a/tools/testing/selftests/filesystems/mntns_cleanup/Makefile b/tools/testing/selftests/filesystems/mntns_cleanup/Makefile deleted file mode 100644 index 0e09e7030a5c..000000000000 --- a/tools/testing/selftests/filesystems/mntns_cleanup/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -TEST_GEN_PROGS := mntns_cleanup_test - -CFLAGS += -Wall -O2 -g $(KHDR_INCLUDES) - -include ../../lib.mk diff --git a/tools/testing/selftests/filesystems/mntns_cleanup/mntns_cleanup_test.c b/tools/testing/selftests/filesystems/mntns_cleanup/mntns_cleanup_test.c deleted file mode 100644 index 5209712568b1..000000000000 --- a/tools/testing/selftests/filesystems/mntns_cleanup/mntns_cleanup_test.c +++ /dev/null @@ -1,58 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include - -#include "../../kselftest_harness.h" - -FIXTURE(mntns_cleanup) { -}; - -FIXTURE_SETUP(mntns_cleanup) -{ - if (geteuid() != 0) - SKIP(return, "test requires CAP_SYS_ADMIN"); - - ASSERT_EQ(unshare(CLONE_NEWNS), 0); - ASSERT_EQ(mount("", "/", NULL, MS_REC | MS_PRIVATE, NULL), 0); - - rmdir("/mnt_dir"); - ASSERT_EQ(mkdir("/mnt_dir", 0755), 0); - ASSERT_EQ(mount("tmpfs", "/mnt_dir", "tmpfs", 0, NULL), 0); - ASSERT_EQ(mkdir("/mnt_dir/hidden", 0755), 0); - ASSERT_EQ(mkdir("/mnt_dir/hidden/secret", 0755), 0); - ASSERT_EQ(mount("tmpfs", "/mnt_dir/hidden", "tmpfs", 0, NULL), 0); -} - -FIXTURE_TEARDOWN(mntns_cleanup) -{ -} - -/* Mounts must stay connected when a mount namespace is cleaned up. */ -TEST_F(mntns_cleanup, keeps_mounts_connected) -{ - int fd, sfd, err; - - fd = open("/mnt_dir", O_PATH | O_DIRECTORY | O_CLOEXEC); - ASSERT_GE(fd, 0); - - /* Destroy the namespace; the fd keeps /mnt_dir alive. */ - ASSERT_EQ(unshare(CLONE_NEWNS), 0); - - sfd = openat(fd, "hidden/secret", O_RDONLY); - err = errno; - if (sfd >= 0) - close(sfd); - close(fd); - - ASSERT_LT(sfd, 0) - TH_LOG("mount namespace teardown revealed what the overmount covered"); - ASSERT_EQ(err, ENOENT); -} - -TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_non_uniq_symbol.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_non_uniq_symbol.tc index bc9514428dba..07b1177c1634 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_non_uniq_symbol.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_non_uniq_symbol.tc @@ -6,7 +6,7 @@ SYMBOL='name_show' # We skip this test on kernel where SYMBOL is unique or does not exist. -if [ "$(grep -c -E "[[:alnum:]]+ t ${SYMBOL}" /proc/kallsyms)" -le '1' ]; then +if [ "$(grep -c -E "[[:alnum:]]+ t ${SYMBOL}$" /proc/kallsyms)" -le '1' ]; then exit_unsupported fi diff --git a/tools/testing/selftests/hid/hid_bpf.c b/tools/testing/selftests/hid/hid_bpf.c index 7ab86296ff23..8c41ebfb85e4 100644 --- a/tools/testing/selftests/hid/hid_bpf.c +++ b/tools/testing/selftests/hid/hid_bpf.c @@ -5,7 +5,7 @@ #include struct hid_hw_request_syscall_args { - __u8 data[10]; + __u8 data[MAX_BUF_SIZE]; unsigned int hid; int retval; size_t size; @@ -54,11 +54,27 @@ FIXTURE_TEARDOWN(hid_bpf) { hid_bpf_teardown(_metadata, self, variant); \ } while (0) +FIXTURE_VARIANT(hid_bpf) { + __u8 *rdesc; + size_t rdesc_size; +}; + +FIXTURE_VARIANT_ADD(hid_bpf, numbered) { + .rdesc = rdesc, + .rdesc_size = sizeof(rdesc), +}; + +FIXTURE_VARIANT_ADD(hid_bpf, unnumbered) { + .rdesc = fido2_rdesc, + .rdesc_size = sizeof(fido2_rdesc), +}; + FIXTURE_SETUP(hid_bpf) { int err; - err = setup_uhid(_metadata, &self->hid, BUS_USB, 0x0001, 0x0a36, rdesc, sizeof(rdesc)); + err = setup_uhid(_metadata, &self->hid, BUS_USB, 0x0001, 0x0a36, + variant->rdesc, variant->rdesc_size); ASSERT_OK(err); } @@ -175,7 +191,7 @@ TEST_F(hid_bpf, raw_event) const struct test_program progs[] = { { .name = "hid_first_event" }, }; - __u8 buf[10] = {0}; + __u8 buf[MAX_BUF_SIZE] = {0}; int err; LOAD_PROGRAMS(progs); @@ -226,7 +242,7 @@ TEST_F(hid_bpf, subprog_raw_event) const struct test_program progs[] = { { .name = "hid_subprog_first_event" }, }; - __u8 buf[10] = {0}; + __u8 buf[MAX_BUF_SIZE] = {0}; int err; LOAD_PROGRAMS(progs); @@ -284,7 +300,7 @@ TEST_F(hid_bpf, test_attach_detach) { .name = "hid_second_event" }, }; struct bpf_link *link; - __u8 buf[10] = {0}; + __u8 buf[MAX_BUF_SIZE] = {0}; int err, link_fd; LOAD_PROGRAMS(progs); @@ -369,7 +385,7 @@ TEST_F(hid_bpf, test_hid_change_report) const struct test_program progs[] = { { .name = "hid_change_report_id" }, }; - __u8 buf[10] = {0}; + __u8 buf[MAX_BUF_SIZE] = {0}; int err; LOAD_PROGRAMS(progs); @@ -396,21 +412,24 @@ TEST_F(hid_bpf, test_hid_user_input_report_call) { struct hid_hw_request_syscall_args args = { .retval = -1, - .size = 10, + .size = MAX_BUF_SIZE, }; DECLARE_LIBBPF_OPTS(bpf_test_run_opts, tattrs, .ctx_in = &args, .ctx_size_in = sizeof(args), ); - __u8 buf[10] = {0}; + __u8 buf[MAX_BUF_SIZE] = {0}; int err, prog_fd; LOAD_BPF; args.hid = self->hid.hid_id; args.data[0] = 1; /* report ID */ - args.data[1] = 2; /* report ID */ - args.data[2] = 42; /* report ID */ + args.data[1] = 2; + args.data[2] = 42; + + if (variant->rdesc == fido2_rdesc) + args.data[0] = 0; prog_fd = bpf_program__fd(self->skel->progs.hid_user_input_report); @@ -428,8 +447,13 @@ TEST_F(hid_bpf, test_hid_user_input_report_call) /* read the data from hidraw */ memset(buf, 0, sizeof(buf)); err = read(self->hidraw_fd, buf, sizeof(buf)); - ASSERT_EQ(err, 6) TH_LOG("read_hidraw"); - ASSERT_EQ(buf[0], 1); + if (variant->rdesc == rdesc) { + ASSERT_EQ(err, 6) TH_LOG("read_hidraw"); + } else { + ASSERT_EQ(err, 64) + TH_LOG("read_hidraw"); + } + ASSERT_EQ(buf[0], args.data[0]); ASSERT_EQ(buf[1], 2); ASSERT_EQ(buf[2], 42); } @@ -442,7 +466,7 @@ TEST_F(hid_bpf, test_hid_user_output_report_call) { struct hid_hw_request_syscall_args args = { .retval = -1, - .size = 10, + .size = MAX_BUF_SIZE, }; DECLARE_LIBBPF_OPTS(bpf_test_run_opts, tattrs, .ctx_in = &args, @@ -455,8 +479,11 @@ TEST_F(hid_bpf, test_hid_user_output_report_call) args.hid = self->hid.hid_id; args.data[0] = 1; /* report ID */ - args.data[1] = 2; /* report ID */ - args.data[2] = 42; /* report ID */ + args.data[1] = 2; + args.data[2] = 42; + + if (variant->rdesc == fido2_rdesc) + args.data[0] = 0; prog_fd = bpf_program__fd(self->skel->progs.hid_user_output_report); @@ -472,9 +499,14 @@ TEST_F(hid_bpf, test_hid_user_output_report_call) ASSERT_OK(err) TH_LOG("error while calling bpf_prog_test_run_opts"); ASSERT_OK(cond_err) TH_LOG("error while calling waiting for the condition"); - ASSERT_EQ(args.retval, 3); + if (variant->rdesc == rdesc) { + ASSERT_EQ(args.retval, 3); + } else if (variant->rdesc == fido2_rdesc) { + ASSERT_EQ(args.retval, 65) + TH_LOG("report size error, should have 64 + 1 extra byte for the report ID 0"); + } - ASSERT_EQ(output_report[0], 1); + ASSERT_EQ(output_report[0], args.data[0]); ASSERT_EQ(output_report[1], 2); ASSERT_EQ(output_report[2], 42); @@ -491,7 +523,7 @@ TEST_F(hid_bpf, test_hid_user_raw_request_call) .retval = -1, .type = HID_FEATURE_REPORT, .request_type = HID_REQ_GET_REPORT, - .size = 10, + .size = MAX_BUF_SIZE, }; DECLARE_LIBBPF_OPTS(bpf_test_run_opts, tattrs, .ctx_in = &args, @@ -514,6 +546,37 @@ TEST_F(hid_bpf, test_hid_user_raw_request_call) ASSERT_EQ(args.data[1], 2); } +/* + * Call hid_hw_raw_request against the given uhid device, + * check that the program is called and does the expected. + */ +TEST_F(hid_bpf, test_hid_user_raw_request_call_eio) +{ + struct hid_hw_request_syscall_args args = { + .retval = -1, + .type = HID_FEATURE_REPORT, + .request_type = HID_REQ_GET_PROTOCOL, + .size = MAX_BUF_SIZE, + }; + DECLARE_LIBBPF_OPTS(bpf_test_run_opts, tattrs, + .ctx_in = &args, + .ctx_size_in = sizeof(args), + ); + int err, prog_fd; + + LOAD_BPF; + + args.hid = self->hid.hid_id; + args.data[0] = 1; /* report ID */ + + prog_fd = bpf_program__fd(self->skel->progs.hid_user_raw_request); + + err = bpf_prog_test_run_opts(prog_fd, &tattrs); + ASSERT_OK(err) TH_LOG("error while calling bpf_prog_test_run_opts"); + + ASSERT_EQ(args.retval, -EIO); +} + /* * Call hid_hw_raw_request against the given uhid device, * check that the program is called and prevents the @@ -524,7 +587,7 @@ TEST_F(hid_bpf, test_hid_filter_raw_request_call) const struct test_program progs[] = { { .name = "hid_test_filter_raw_request" }, }; - __u8 buf[10] = {0}; + __u8 buf[MAX_BUF_SIZE] = {0}; int err; LOAD_PROGRAMS(progs); @@ -577,7 +640,7 @@ TEST_F(hid_bpf, test_hid_change_raw_request_call) const struct test_program progs[] = { { .name = "hid_test_hidraw_raw_request" }, }; - __u8 buf[10] = {0}; + __u8 buf[MAX_BUF_SIZE] = {0}; int err; LOAD_PROGRAMS(progs); @@ -603,7 +666,7 @@ TEST_F(hid_bpf, test_hid_infinite_loop_raw_request_call) const struct test_program progs[] = { { .name = "hid_test_infinite_loop_raw_request" }, }; - __u8 buf[10] = {0}; + __u8 buf[MAX_BUF_SIZE] = {0}; int err; LOAD_PROGRAMS(progs); @@ -626,7 +689,7 @@ TEST_F(hid_bpf, test_hid_filter_output_report_call) const struct test_program progs[] = { { .name = "hid_test_filter_output_report" }, }; - __u8 buf[10] = {0}; + __u8 buf[MAX_BUF_SIZE] = {0}; int err; LOAD_PROGRAMS(progs); @@ -679,7 +742,7 @@ TEST_F(hid_bpf, test_hid_change_output_report_call) const struct test_program progs[] = { { .name = "hid_test_hidraw_output_report" }, }; - __u8 buf[10] = {0}; + __u8 buf[MAX_BUF_SIZE] = {0}; int err; LOAD_PROGRAMS(progs); @@ -703,7 +766,7 @@ TEST_F(hid_bpf, test_hid_infinite_loop_output_report_call) const struct test_program progs[] = { { .name = "hid_test_infinite_loop_output_report" }, }; - __u8 buf[10] = {0}; + __u8 buf[MAX_BUF_SIZE] = {0}; int err; LOAD_PROGRAMS(progs); @@ -729,7 +792,7 @@ TEST_F(hid_bpf, test_multiply_events_wq) const struct test_program progs[] = { { .name = "hid_test_multiply_events_wq" }, }; - __u8 buf[10] = {0}; + __u8 buf[MAX_BUF_SIZE] = {0}; int err; LOAD_PROGRAMS(progs); @@ -767,7 +830,7 @@ TEST_F(hid_bpf, test_multiply_events) const struct test_program progs[] = { { .name = "hid_test_multiply_events" }, }; - __u8 buf[10] = {0}; + __u8 buf[MAX_BUF_SIZE] = {0}; int err; LOAD_PROGRAMS(progs); @@ -801,7 +864,7 @@ TEST_F(hid_bpf, test_hid_infinite_loop_input_report_call) const struct test_program progs[] = { { .name = "hid_test_infinite_loop_input_report" }, }; - __u8 buf[10] = {0}; + __u8 buf[MAX_BUF_SIZE] = {0}; int err; LOAD_PROGRAMS(progs); @@ -855,7 +918,7 @@ TEST_F(hid_bpf, test_hid_attach_flags) .insert_head = 0, }, }; - __u8 buf[10] = {0}; + __u8 buf[MAX_BUF_SIZE] = {0}; int err; LOAD_PROGRAMS(progs); @@ -886,6 +949,9 @@ TEST_F(hid_bpf, test_rdesc_fixup) }; int err, desc_size; + if (variant->rdesc != rdesc) + SKIP(return, "not compatible report descriptor"); + LOAD_PROGRAMS(progs); /* check that hid_rdesc_fixup() was executed */ diff --git a/tools/testing/selftests/hid/hid_common.h b/tools/testing/selftests/hid/hid_common.h index e3b267446fa0..b7890ba2878f 100644 --- a/tools/testing/selftests/hid/hid_common.h +++ b/tools/testing/selftests/hid/hid_common.h @@ -13,6 +13,7 @@ #include #define SHOW_UHID_DEBUG 0 +#define MAX_BUF_SIZE 128 #define min(a, b) \ ({ __typeof__(a) _a = (a); \ @@ -97,6 +98,28 @@ static unsigned char rdesc[] = { static __u8 feature_data[] = { 1, 2 }; +static __maybe_unused unsigned char fido2_rdesc[] = { + 0x06, 0xd0, 0xf1, /* Usage Page (FIDO Alliance) */ + 0x09, 0x01, /* Usage (U2F Authenticator Device) */ + 0xa1, 0x01, /* Collection (Application) */ + 0x09, 0x20, /* Usage (Input Report Data) */ + 0x15, 0x00, /* Logical Minimum (0) */ + 0x26, 0xff, 0x00, /* Logical Maximum (255) */ + 0x75, 0x08, /* Report Size (8) */ + 0x95, 0x40, /* Report Count (64) */ + 0x81, 0x02, /* Input (Data,Var,Abs) */ + 0x09, 0x21, /* Usage (Output Report Data) */ + 0x15, 0x00, /* Logical Minimum (0) */ + 0x26, 0xff, 0x00, /* Logical Maximum (255) */ + 0x75, 0x08, /* Report Size (8) */ + 0x95, 0x40, /* Report Count (64) */ + 0x91, 0x02, /* Output (Data,Var,Abs) */ + 0x06, 0x00, 0xff, /* Usage Page (Vendor Defined Page 1) */ + 0x09, 0x22, /* Usage (Vendor Usage 0x22) */ + 0xb1, 0x02, /* Feature (Data,Var,Abs) */ + 0xc0, /* End Collection */ +}; + #define ASSERT_OK(data) ASSERT_FALSE(data) #define ASSERT_OK_PTR(ptr) ASSERT_NE(NULL, ptr) @@ -110,7 +133,7 @@ static pthread_cond_t uhid_started = PTHREAD_COND_INITIALIZER; static pthread_mutex_t uhid_output_mtx = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t uhid_output_cond = PTHREAD_COND_INITIALIZER; -static unsigned char output_report[10]; +static unsigned char output_report[MAX_BUF_SIZE]; /* no need to protect uhid_stopped, only one thread accesses it */ static bool uhid_stopped; diff --git a/tools/testing/selftests/hid/progs/hid.c b/tools/testing/selftests/hid/progs/hid.c index 361dc7eaad22..48aa8088cc53 100644 --- a/tools/testing/selftests/hid/progs/hid.c +++ b/tools/testing/selftests/hid/progs/hid.c @@ -98,7 +98,7 @@ struct hid_bpf_ops change_report_id = { struct hid_hw_request_syscall_args { /* data needs to come at offset 0 so we can use it in calls */ - __u8 data[10]; + __u8 data[128]; unsigned int hid; int retval; size_t size; diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index 96bab7002d39..f43549a56f0b 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -103,6 +103,7 @@ TEST_GEN_PROGS_x86 += x86/nested_tdp_fault_test TEST_GEN_PROGS_x86 += x86/nested_tsc_adjust_test TEST_GEN_PROGS_x86 += x86/nested_tsc_scaling_test TEST_GEN_PROGS_x86 += x86/nested_vmsave_vmload_test +TEST_GEN_PROGS_x86 += x86/nested_x2apic_test TEST_GEN_PROGS_x86 += x86/platform_info_test TEST_GEN_PROGS_x86 += x86/pmu_counters_test TEST_GEN_PROGS_x86 += x86/pmu_event_filter_test @@ -189,11 +190,13 @@ TEST_GEN_PROGS_arm64 += arm64/stage2_block_transitions TEST_GEN_PROGS_arm64 += arm64/vcpu_width_config TEST_GEN_PROGS_arm64 += arm64/vgic_init TEST_GEN_PROGS_arm64 += arm64/vgic_irq +TEST_GEN_PROGS_arm64 += arm64/vgic_its_save TEST_GEN_PROGS_arm64 += arm64/vgic_lpi_stress TEST_GEN_PROGS_arm64 += arm64/vgic_v5 TEST_GEN_PROGS_arm64 += arm64/vpmu_counter_access TEST_GEN_PROGS_arm64 += arm64/no-vgic TEST_GEN_PROGS_arm64 += arm64/idreg-idst +TEST_GEN_PROGS_arm64 += arm64/hidden_features TEST_GEN_PROGS_arm64 += arm64/kvm-uuid TEST_GEN_PROGS_arm64 += access_tracking_perf_test TEST_GEN_PROGS_arm64 += arch_timer diff --git a/tools/testing/selftests/kvm/arm64/hidden_features.c b/tools/testing/selftests/kvm/arm64/hidden_features.c new file mode 100644 index 000000000000..194d746e7605 --- /dev/null +++ b/tools/testing/selftests/kvm/arm64/hidden_features.c @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * hidden_features - Check that a feature's instruction runs in the guest when + * its ID register field is advertised, and is UNDEFINED when userspace clears + * the field. + * + * Copyright (c) 2026 Google LLC + * Author: Fuad Tabba + */ +#include "kvm_util.h" +#include "processor.h" +#include "test_util.h" + +static volatile bool undef; + +static void guest_tlbi_os(void) +{ + /* tlbi vmalle1os */ + asm volatile("sys #0, c8, c1, #0\n\tdsb ish\n\tisb" ::: "memory"); +} + +static void guest_mops(void) +{ + register u64 *d asm("x0"); + register u64 n asm("x1"); + register u64 s asm("x2"); + u64 buf[8]; + + d = buf; + n = sizeof(buf); + s = 0; + /* setp [x0]!, x1!, x2; setm; sete */ + asm volatile(".inst 0x19c20420\n\t.inst 0x19c24420\n\t.inst 0x19c28420" + : "+r"(d), "+r"(n) : "r"(s) : "cc", "memory"); +} + +static void guest_tcr2(void) +{ + read_sysreg_s(SYS_TCR2_EL1); +} + +static void guest_fpmr(void) +{ + read_sysreg_s(SYS_FPMR); +} + +struct feature { + const char *name; + u64 id_reg; + u64 mask; + u8 shift; + u64 min; + void (*insn)(void); + bool (*trappable)(struct kvm_vcpu *vcpu); +}; + +/* Without FGT, KVM traps a hidden TLBI OS only through HCR_EL2.TTLBOS (FEAT_EVT2). */ +static bool tlbi_os_trappable(struct kvm_vcpu *vcpu) +{ + u64 mmfr0 = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64MMFR0_EL1)); + u64 mmfr2 = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64MMFR2_EL1)); + + return SYS_FIELD_GET(ID_AA64MMFR0_EL1, FGT, mmfr0) >= ID_AA64MMFR0_EL1_FGT_IMP || + SYS_FIELD_GET(ID_AA64MMFR2_EL1, EVT, mmfr2) >= ID_AA64MMFR2_EL1_EVT_TTLBxS; +} + +#define FEATURE(n, reg, field, min_val, fn, trap) \ +{ \ + .name = n, \ + .id_reg = SYS_##reg, \ + .mask = reg##_##field##_MASK, \ + .shift = reg##_##field##_SHIFT, \ + .min = reg##_##field##_##min_val, \ + .insn = fn, \ + .trappable = trap, \ +} + +static const struct feature features[] = { + FEATURE("TLBI OS", ID_AA64ISAR0_EL1, TLB, OS, guest_tlbi_os, tlbi_os_trappable), + FEATURE("MOPS", ID_AA64ISAR2_EL1, MOPS, IMP, guest_mops, NULL), + FEATURE("TCR2_EL1", ID_AA64MMFR3_EL1, TCRX, IMP, guest_tcr2, NULL), + FEATURE("FPMR", ID_AA64PFR2_EL1, FPMR, IMP, guest_fpmr, NULL), +}; + +static void guest_code(const struct feature *feat) +{ + undef = false; + feat->insn(); + GUEST_SYNC(undef); + GUEST_DONE(); +} + +static void guest_undef_handler(struct ex_regs *regs) +{ + undef = true; + regs->pc += 4; +} + +static bool run(const struct feature *feat, bool hide) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + struct ucall uc; + bool got = false; + u64 val; + + vm = vm_create_with_one_vcpu(&vcpu, (void *)guest_code); + vm_init_descriptor_tables(vm); + vcpu_init_descriptor_tables(vcpu); + vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT, ESR_ELx_EC_UNKNOWN, guest_undef_handler); + vcpu_args_set(vcpu, 1, feat); + + if (hide) { + val = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(feat->id_reg)); + vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(feat->id_reg), val & ~feat->mask); + } + + for (;;) { + vcpu_run(vcpu); + switch (get_ucall(vcpu, &uc)) { + case UCALL_SYNC: + got = uc.args[1]; + break; + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + break; + case UCALL_DONE: + kvm_vm_free(vm); + return got; + default: + TEST_FAIL("Unknown ucall %lu", uc.cmd); + } + } +} + +static void probe_feature(const struct feature *feat, bool *present, bool *trappable) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + u64 val; + + vm = vm_create_with_one_vcpu(&vcpu, NULL); + val = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(feat->id_reg)); + *present = ((val & feat->mask) >> feat->shift) >= feat->min; + *trappable = !feat->trappable || feat->trappable(vcpu); + kvm_vm_free(vm); +} + +int main(void) +{ + const struct feature *feat; + bool present, trappable; + int i; + + test_disable_default_vgic(); + + ksft_print_header(); + ksft_set_plan(ARRAY_SIZE(features) * 2); + + for (i = 0; i < ARRAY_SIZE(features); i++) { + feat = &features[i]; + + probe_feature(feat, &present, &trappable); + if (!present) { + ksft_test_result_skip("%s advertised, not supported\n", feat->name); + ksft_test_result_skip("%s hidden, not supported\n", feat->name); + continue; + } + + if (run(feat, false)) + ksft_test_result_fail("%s advertised, UNDEF\n", feat->name); + else + ksft_test_result_pass("%s advertised\n", feat->name); + + if (!trappable) + ksft_test_result_skip("%s hidden, not trappable\n", feat->name); + else if (run(feat, true)) + ksft_test_result_pass("%s hidden\n", feat->name); + else + ksft_test_result_fail("%s hidden, no UNDEF\n", feat->name); + } + + ksft_finished(); +} diff --git a/tools/testing/selftests/kvm/arm64/smccc_filter.c b/tools/testing/selftests/kvm/arm64/smccc_filter.c index 21e41880261b..a41ed3e016ba 100644 --- a/tools/testing/selftests/kvm/arm64/smccc_filter.c +++ b/tools/testing/selftests/kvm/arm64/smccc_filter.c @@ -140,6 +140,10 @@ static void test_invalid_nr_functions(void) TEST_ASSERT(r < 0 && errno == EINVAL, "Attempt to filter 0 functions should return EINVAL"); + r = __set_smccc_filter(vm, 0, 0, KVM_SMCCC_FILTER_DENY); + TEST_ASSERT(r < 0 && errno == EINVAL, + "Attempt to filter 0 functions at base 0 should return EINVAL"); + kvm_vm_free(vm); } diff --git a/tools/testing/selftests/kvm/arm64/vgic_its_save.c b/tools/testing/selftests/kvm/arm64/vgic_its_save.c new file mode 100644 index 000000000000..864da01539f3 --- /dev/null +++ b/tools/testing/selftests/kvm/arm64/vgic_its_save.c @@ -0,0 +1,441 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * vgic_its_save - KVM_DEV_ARM_ITS_SAVE_TABLES against tables a guest broke. + * + * Both cases are reachable by a guest on its own, and neither may fail a save + * that userspace has to be able to issue: + * + * - Changing GITS_BASER drops the collections it described, so the save + * writes nothing but the terminating invalid entry. + * - A device the device table can no longer address is skipped, and the saved + * DTE chain skips it too rather than pointing at an entry never written. + * + * Both cases then reset and restore, which is what the save exists for. + * + * Copyright (c) 2026 Google LLC + * Author: Fuad Tabba + */ + +#include +#include +#include +#include + +#include "kvm_util.h" +#include "gic.h" +#include "gic_v3.h" +#include "gic_v3_its.h" +#include "processor.h" +#include "ucall.h" +#include "vgic.h" + +#define TEST_MEMSLOT_INDEX 1 + +/* All three ITS table entry sizes are 8 bytes in ABI 0. */ +#define ESZ 8 +#define ENTRIES_PER_PAGE (SZ_64K / ESZ) + +/* CTE and DTE layout, mirroring KVM's KVM_ITS_* in arch/arm64/kvm/vgic/vgic.h */ +#define CTE_VALID_MASK BIT_ULL(63) +#define DTE_VALID_MASK BIT_ULL(63) +#define DTE_NEXT_SHIFT 49 +#define DTE_NEXT_MASK GENMASK_ULL(62, 49) + +/* L1 entry of an indirect table: valid bit plus a 64K aligned L2 address. */ +#define L1E_VALID_MASK BIT_ULL(63) +#define L1E_ADDR_MASK GENMASK_ULL(51, 16) + +#define GITS_BASER_PAGES_MASK GENMASK_ULL(7, 0) + +#define POISON 0xdeadbeefdeadbeefULL + +/* The collection table starts at two pages and is shrunk to one. */ +#define COLL_TBL_PAGES 2 +#define COLL_TBL_SZ (COLL_TBL_PAGES * SZ_64K) + +/* One more collection than the shrunken table can hold. */ +#define NR_COLLECTIONS (ENTRIES_PER_PAGE + 1) + +/* Two devices, one per L2 block of the indirect device table. */ +#define DEVICE_A_ID 0 +#define DEVICE_B_ID ENTRIES_PER_PAGE + +/* + * its_send_mapd_cmd() encodes ilog2(itt_size) - 1 as num_eventid_bits, and + * vgic_its_restore_itt() scans BIT_ULL(num_eventid_bits) * ESZ, so the size + * handed to MAPD has to match the ITT allocated for it. + */ +#define ITT_EVENTID_BITS 13 +#define ITT_MAPD_SIZE BIT_ULL(ITT_EVENTID_BITS + 1) +#define ITT_SZ (BIT_ULL(ITT_EVENTID_BITS) * ESZ) + +static struct kvm_vm *vm; +static struct kvm_vcpu *vcpu; +static int its_fd; +static gpa_t gpa_base; + +static struct test_data { + gpa_t device_table; + gpa_t collection_table; + gpa_t cmdq_base; + void *cmdq_base_va; + + gpa_t lpi_prop_table; + gpa_t lpi_pend_table; + + void *device_l1_va; + gpa_t device_l2[2]; + gpa_t itt_tables; +} test_data; + +static unsigned long its_baser_offset(unsigned int type) +{ + int i; + + for (i = 0; i < GITS_BASER_NR_REGS; i++) { + unsigned long offset = GITS_BASER + (i * sizeof(u64)); + u64 baser = readq_relaxed(GITS_BASE_GVA + offset); + + if (GITS_BASER_TYPE(baser) == type) + return offset; + } + + GUEST_FAIL("Couldn't find an ITS BASER of type %u", type); + return -1; +} + +static void its_set_enable(bool enable) +{ + u32 ctlr = readl_relaxed(GITS_BASE_GVA + GITS_CTLR); + + if (enable) + ctlr |= GITS_CTLR_ENABLE; + else + ctlr &= ~GITS_CTLR_ENABLE; + + writel_relaxed(ctlr, GITS_BASE_GVA + GITS_CTLR); +} + +/* + * Shrink the collection table to a single page, leaving VALID set. BASER + * writes are ignored while the ITS is enabled. + */ +static void guest_shrink_coll_table(void) +{ + unsigned long offset = its_baser_offset(GITS_BASER_TYPE_COLLECTION); + u64 baser; + + its_set_enable(false); + + baser = readq_relaxed(GITS_BASE_GVA + offset); + baser &= ~GITS_BASER_PAGES_MASK; + writeq_relaxed(baser, GITS_BASE_GVA + offset); +} + +static void guest_baser_change(void) +{ + u32 coll_id; + + gic_init(GIC_V3, 1); + gic_rdist_enable_lpis(test_data.lpi_prop_table, SZ_64K, + test_data.lpi_pend_table); + + its_init(test_data.collection_table, COLL_TBL_SZ, + test_data.device_table, SZ_64K, + test_data.cmdq_base, SZ_64K); + + for (coll_id = 0; coll_id < NR_COLLECTIONS; coll_id++) + its_send_mapc_cmd(test_data.cmdq_base_va, 0, coll_id, true); + + guest_shrink_coll_table(); + + GUEST_DONE(); +} + +/* Turn the already installed device table into an indirect one. */ +static void guest_make_device_table_indirect(void) +{ + unsigned long offset = its_baser_offset(GITS_BASER_TYPE_DEVICE); + u64 baser; + + its_set_enable(false); + + baser = readq_relaxed(GITS_BASE_GVA + offset); + writeq_relaxed(baser | GITS_BASER_INDIRECT, GITS_BASE_GVA + offset); + + its_set_enable(true); +} + +static void guest_unreachable_device(void) +{ + u64 *l1; + + gic_init(GIC_V3, 1); + gic_rdist_enable_lpis(test_data.lpi_prop_table, SZ_64K, + test_data.lpi_pend_table); + + its_init(test_data.collection_table, SZ_64K, + test_data.device_table, SZ_64K, + test_data.cmdq_base, SZ_64K); + + guest_make_device_table_indirect(); + + /* Both L2 blocks present, so both MAPDs are in range. */ + l1 = test_data.device_l1_va; + l1[0] = L1E_VALID_MASK | (test_data.device_l2[0] & L1E_ADDR_MASK); + l1[1] = L1E_VALID_MASK | (test_data.device_l2[1] & L1E_ADDR_MASK); + + its_send_mapd_cmd(test_data.cmdq_base_va, DEVICE_A_ID, + test_data.itt_tables, ITT_MAPD_SIZE, true); + its_send_mapd_cmd(test_data.cmdq_base_va, DEVICE_B_ID, + test_data.itt_tables + ITT_SZ, ITT_MAPD_SIZE, true); + + /* + * Drop the block holding device B. No ITS command and no GITS_BASER + * write is involved, so nothing tells KVM the device is now + * unreachable. + */ + l1[1] = 0; + + GUEST_DONE(); +} + +static void run_guest(void) +{ + struct ucall uc; + + vcpu_run(vcpu); + switch (get_ucall(vcpu, &uc)) { + case UCALL_DONE: + break; + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + break; + default: + TEST_FAIL("Unexpected ucall: %lu", uc.cmd); + } +} + +static int save_tables(void) +{ + return __kvm_device_attr_set(its_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_ITS_SAVE_TABLES, NULL); +} + +static u64 its_reg_get(unsigned long offset) +{ + u64 val; + + kvm_device_attr_get(its_fd, KVM_DEV_ARM_VGIC_GRP_ITS_REGS, offset, + &val); + return val; +} + +static void its_reg_set(unsigned long offset, u64 val) +{ + kvm_device_attr_set(its_fd, KVM_DEV_ARM_VGIC_GRP_ITS_REGS, offset, + &val); +} + +/* + * What a migration target does with the saved tables, in the order + * Documentation/virt/kvm/devices/arm-vgic-its.rst gives: the GITS_ registers + * first, then the tables. The reset in between clears GITS_BASER.Valid, + * which is why the registers have to be written back before the restore. + */ +static void reset_and_restore_tables(void) +{ + u64 baser[GITS_BASER_NR_REGS]; + int ret, i; + + for (i = 0; i < GITS_BASER_NR_REGS; i++) + baser[i] = its_reg_get(GITS_BASER + (i * sizeof(u64))); + + ret = __kvm_device_attr_set(its_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_ITS_CTRL_RESET, NULL); + TEST_ASSERT(!ret, "Expected the reset to succeed, got ret %d errno %d", + ret, errno); + + for (i = 0; i < GITS_BASER_NR_REGS; i++) + its_reg_set(GITS_BASER + (i * sizeof(u64)), baser[i]); + + ret = __kvm_device_attr_set(its_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_ITS_RESTORE_TABLES, NULL); + TEST_ASSERT(!ret, "Expected the restore to succeed, got ret %d errno %d", + ret, errno); +} + +static void poison_range(gpa_t base, size_t size) +{ + u64 *entry = addr_gpa2hva(vm, base); + size_t i; + + for (i = 0; i < size / ESZ; i++) + entry[i] = POISON; +} + +static void setup_memslot(size_t sz) +{ + size_t pages = sz / vm->page_size; + + gpa_base = ((vm_compute_max_gfn(vm) + 1) * vm->page_size) - sz; + vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, gpa_base, + TEST_MEMSLOT_INDEX, pages, 0); +} + +static gpa_t alloc_64k(size_t nr) +{ + size_t pages_per_64k = vm_calc_num_guest_pages(vm->mode, SZ_64K); + gpa_t gpa = vm_phy_pages_alloc(vm, nr * pages_per_64k, gpa_base, + TEST_MEMSLOT_INDEX); + + TEST_ASSERT(IS_ALIGNED(gpa, SZ_64K), + "Allocation at 0x%lx is not 64K aligned, GITS_BASER cannot address it", + gpa); + return gpa; +} + +static void map_to_guest(gpa_t gpa, size_t nr) +{ + size_t pages_per_64k = vm_calc_num_guest_pages(vm->mode, SZ_64K); + + virt_map(vm, gpa, gpa, nr * pages_per_64k); +} + +static void setup_common(void) +{ + test_data.cmdq_base = alloc_64k(1); + map_to_guest(test_data.cmdq_base, 1); + test_data.cmdq_base_va = (void *)test_data.cmdq_base; + + test_data.lpi_prop_table = alloc_64k(1); + test_data.lpi_pend_table = alloc_64k(1); +} + +static void teardown(void) +{ + close(its_fd); + kvm_vm_free(vm); + memset(&test_data, 0, sizeof(test_data)); +} + +/* + * A GITS_BASER write that changes the table drops the collections it + * described. The save then has an empty list, so it writes the terminating + * invalid entry and nothing else. + */ +static void test_baser_change_drops_collections(void) +{ + u64 *cte; + int ret, i; + + pr_info("Testing that a GITS_BASER change drops the collections\n"); + + vm = vm_create_with_one_vcpu(&vcpu, guest_baser_change); + setup_memslot((4 + COLL_TBL_PAGES) * SZ_64K); + its_fd = vgic_its_setup(vm); + + test_data.device_table = alloc_64k(1); + test_data.collection_table = alloc_64k(COLL_TBL_PAGES); + setup_common(); + + sync_global_to_guest(vm, test_data); + run_guest(); + + /* Anything KVM writes is then the only thing that changed. */ + poison_range(test_data.collection_table, COLL_TBL_SZ); + + ret = save_tables(); + TEST_ASSERT(!ret, "Expected the save to succeed, got %d errno %d", + ret, errno); + + cte = addr_gpa2hva(vm, test_data.collection_table); + + /* + * Finding the terminator at the head of the table is also what proves + * the reads below landed in the saved table rather than elsewhere. + */ + TEST_ASSERT(le64toh(cte[0]) == 0, + "CTE 0: expected the terminating invalid entry, got 0x%llx", + (unsigned long long)le64toh(cte[0])); + + for (i = 1; i < COLL_TBL_SZ / ESZ; i++) + TEST_ASSERT(cte[i] == POISON, + "CTE %d: expected it untouched, got 0x%llx", + i, (unsigned long long)cte[i]); + + reset_and_restore_tables(); + + teardown(); +} + +/* + * A device whose L2 block the guest dropped is skipped by the save, and the + * DTE chain skips it too: left alone, the surviving device would point at an + * entry the save never wrote. + */ +static void test_unreachable_device_skipped(void) +{ + u64 dte; + int ret; + + pr_info("Testing that an unreachable device is skipped by the save\n"); + + vm = vm_create_with_one_vcpu(&vcpu, guest_unreachable_device); + setup_memslot(9 * SZ_64K); + its_fd = vgic_its_setup(vm); + + test_data.device_table = alloc_64k(1); + test_data.collection_table = alloc_64k(1); + test_data.device_l2[0] = alloc_64k(1); + test_data.device_l2[1] = alloc_64k(1); + test_data.itt_tables = alloc_64k(2); + setup_common(); + + map_to_guest(test_data.device_table, 1); + test_data.device_l1_va = (void *)test_data.device_table; + + sync_global_to_guest(vm, test_data); + run_guest(); + + poison_range(test_data.device_l2[0], SZ_64K); + poison_range(test_data.device_l2[1], SZ_64K); + + ret = save_tables(); + TEST_ASSERT(!ret, "Expected the save to succeed, got %d errno %d", + ret, errno); + + dte = le64toh(*(u64 *)addr_gpa2hva(vm, test_data.device_l2[0])); + + /* Device A is still reachable, so it is saved. */ + TEST_ASSERT(dte & DTE_VALID_MASK, + "Device A: expected a valid DTE, got 0x%llx", + (unsigned long long)dte); + + /* + * Device B is the only device after it and was skipped, so nothing + * follows A in the saved chain. + */ + TEST_ASSERT(FIELD_GET(DTE_NEXT_MASK, dte) == 0, + "Device A: expected no next device, got offset %llu", + (unsigned long long)FIELD_GET(DTE_NEXT_MASK, dte)); + + /* And nothing was written into the block the guest dropped. */ + TEST_ASSERT(*(u64 *)addr_gpa2hva(vm, test_data.device_l2[1]) == POISON, + "Device B: expected its entry untouched"); + + reset_and_restore_tables(); + + teardown(); +} + +int main(void) +{ + TEST_REQUIRE(kvm_supports_vgic_v3()); + + test_baser_change_drops_collections(); + test_unreachable_device_skipped(); + + pr_info("All ok!\n"); + return 0; +} diff --git a/tools/testing/selftests/kvm/hardware_disable_test.c b/tools/testing/selftests/kvm/hardware_disable_test.c index 43a36ef3ead8..1c20892d6782 100644 --- a/tools/testing/selftests/kvm/hardware_disable_test.c +++ b/tools/testing/selftests/kvm/hardware_disable_test.c @@ -37,7 +37,7 @@ static void *run_vcpu(void *arg) struct kvm_vcpu *vcpu = arg; struct kvm_run *run = vcpu->run; -#ifndef _GNU_SOURCE +#ifndef __GLIBC__ kvm_sched_setaffinity(0, sizeof(cpu_set_t), &threads_cpu_set); #endif @@ -51,7 +51,7 @@ static void *sleeping_thread(void *arg) { int fd; -#ifndef _GNU_SOURCE +#ifndef __GLIBC__ kvm_sched_setaffinity(0, sizeof(cpu_set_t), &threads_cpu_set); #endif @@ -71,7 +71,7 @@ static void run_test(u32 run) u32 i, j; TEST_ASSERT_EQ(pthread_attr_init(&attr), 0); -#ifdef _GNU_SOURCE +#ifdef __GLIBC__ TEST_ASSERT_EQ(pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &threads_cpu_set), 0); #endif diff --git a/tools/testing/selftests/kvm/steal_time.c b/tools/testing/selftests/kvm/steal_time.c index bc3c62b72c58..785d19f9ee8f 100644 --- a/tools/testing/selftests/kvm/steal_time.c +++ b/tools/testing/selftests/kvm/steal_time.c @@ -27,6 +27,9 @@ static void *st_gva[NR_VCPUS]; static u64 guest_stolen_time[NR_VCPUS]; +static struct kvm_vm *vm_create_steal_time(u32 nr_vcpus, void *guest_code, + struct kvm_vcpu *vcpus[]); + #if defined(__x86_64__) /* steal_time must have 64-byte alignment */ @@ -210,17 +213,14 @@ static void check_steal_time_uapi(void) u64 st_ipa; int ret; - vm = vm_create_with_one_vcpu(&vcpu, NULL); - struct kvm_device_attr dev = { .group = KVM_ARM_VCPU_PVTIME_CTRL, .attr = KVM_ARM_VCPU_PVTIME_IPA, .addr = (u64)&st_ipa, }; + vm = vm_create_steal_time(1, NULL, &vcpu); vcpu_ioctl(vcpu, KVM_HAS_DEVICE_ATTR, &dev); - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, ST_GPA_BASE, 1, 1, 0); - virt_map(vm, ST_GPA_BASE, ST_GPA_BASE, 1); st_ipa = (ulong)ST_GPA_BASE | 1; ret = __vcpu_ioctl(vcpu, KVM_SET_DEVICE_ATTR, &dev); @@ -500,13 +500,27 @@ static void run_vcpu(struct kvm_vcpu *vcpu) } } +static struct kvm_vm *vm_create_steal_time(u32 nr_vcpus, void *guest_code, + struct kvm_vcpu *vcpus[]) +{ + unsigned int gpages; + struct kvm_vm *vm; + + /* Create a VM and an identity mapped memslot for the steal time structure */ + vm = vm_create_with_vcpus(nr_vcpus, guest_code, vcpus); + gpages = vm_calc_num_guest_pages(VM_MODE_DEFAULT, STEAL_TIME_SIZE * nr_vcpus); + vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, ST_GPA_BASE, 1, gpages, 0); + virt_map(vm, ST_GPA_BASE, ST_GPA_BASE, gpages); + + return vm; +} + int main(int ac, char **av) { struct kvm_vcpu *vcpus[NR_VCPUS]; struct kvm_vm *vm; pthread_t thread; cpu_set_t cpuset; - unsigned int gpages; long stolen_time; long run_delay; bool verbose; @@ -517,11 +531,7 @@ int main(int ac, char **av) /* Set CPU affinity so we can force preemption of the VCPU */ cpu = pin_self_to_any_cpu(); - /* Create a VM and an identity mapped memslot for the steal time structure */ - vm = vm_create_with_vcpus(NR_VCPUS, guest_code, vcpus); - gpages = vm_calc_num_guest_pages(VM_MODE_DEFAULT, STEAL_TIME_SIZE * NR_VCPUS); - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, ST_GPA_BASE, 1, gpages, 0); - virt_map(vm, ST_GPA_BASE, ST_GPA_BASE, gpages); + vm = vm_create_steal_time(NR_VCPUS, guest_code, vcpus); ksft_print_header(); TEST_REQUIRE(is_steal_time_supported(vcpus[0])); diff --git a/tools/testing/selftests/kvm/x86/nested_x2apic_test.c b/tools/testing/selftests/kvm/x86/nested_x2apic_test.c new file mode 100644 index 000000000000..ce204ce29a9c --- /dev/null +++ b/tools/testing/selftests/kvm/x86/nested_x2apic_test.c @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include "test_util.h" +#include "kvm_util.h" +#include "processor.h" +#include "vmx.h" +#include "svm_util.h" + +/* + * Use the kernel's posted interrupt vectors to minimize the risk of crashing + * the host if KVM is buggy. Note, the vectors aren't set in stone, ideally + * these will be kept up-to-date if the kernel vectors change, but it's "fine" + * if they are stale. + */ +#define POSTED_INTR_VECTOR 0xf2 +#define POSTED_INTR_WAKEUP_VECTOR 0xf1 +#define POSTED_INTR_NESTED_VECTOR 0xf0 + +static bool inhibit_apicv; +static volatile unsigned int nr_irqs; + +static void guest_irq_handler(struct ex_regs *regs) +{ + nr_irqs++; + x2apic_write_reg(APIC_EOI, 0); +} + +static void l2_guest_code(void) +{ + if (inhibit_apicv) + wrmsr(MSR_IA32_APICBASE, rdmsr(MSR_IA32_APICBASE) & GENMASK_ULL(11, 0)); + + for (;;) { + x2apic_write_reg(APIC_TASKPRI, 0xf0); + GUEST_ASSERT_EQ(x2apic_read_reg(APIC_TASKPRI), 0xf0); + + asm volatile("cpuid" ::: "eax", "ebx", "ecx", "edx"); + } +} + +static void l1_svm_code(struct svm_test_data *svm) +{ + struct vmcb_control_area *ctrl = &svm->vmcb->control; + + generic_svm_setup(svm, l2_guest_code); + ctrl->intercept |= BIT_ULL(INTERCEPT_CPUID) | BIT_ULL(INTERCEPT_MSR_PROT); + + run_guest(svm->vmcb, svm->vmcb_gpa); + GUEST_ASSERT_EQ(ctrl->exit_code, SVM_EXIT_CPUID); + + stgi(); + x2apic_write_reg(APIC_TASKPRI, 0); +} + +static void l1_vmx_code(struct vmx_pages *vmx, struct hyperv_test_pages *hv_pages) +{ + u64 control; + + if (hv_pages) { + wrmsr(HV_X64_MSR_GUEST_OS_ID, HYPERV_LINUX_OS_ID); + enable_vp_assist(hv_pages->vp_assist_gpa, hv_pages->vp_assist); + evmcs_enable(); + } + + GUEST_ASSERT_EQ(prepare_for_vmx_operation(vmx), true); + + if (hv_pages) { + GUEST_ASSERT(load_evmcs(hv_pages)); + current_evmcs->hv_enlightenments_control.msr_bitmap = 1; + } else { + GUEST_ASSERT(load_vmcs(vmx)); + } + + prepare_vmcs(vmx, NULL); + GUEST_ASSERT_EQ(vmwrite(GUEST_RIP, (unsigned long)l2_guest_code), 0); + + control = vmreadz(PIN_BASED_VM_EXEC_CONTROL); + control |= PIN_BASED_EXT_INTR_MASK; + vmwrite(PIN_BASED_VM_EXEC_CONTROL, control); + + control = vmreadz(CPU_BASED_VM_EXEC_CONTROL); + control |= CPU_BASED_USE_MSR_BITMAPS | CPU_BASED_TPR_SHADOW; + GUEST_ASSERT_EQ(vmwrite(CPU_BASED_VM_EXEC_CONTROL, control), 0); + + control = vmreadz(SECONDARY_VM_EXEC_CONTROL); + control |= SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | + SECONDARY_EXEC_APIC_REGISTER_VIRT | + SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY; + control &= (rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2) >> 32); + GUEST_ASSERT_EQ(vmwrite(SECONDARY_VM_EXEC_CONTROL, control), 0); + + GUEST_ASSERT(!vmlaunch()); + GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_CPUID); + GUEST_ASSERT_EQ(vmwrite(GUEST_RIP, + vmreadz(GUEST_RIP) + vmreadz(VM_EXIT_INSTRUCTION_LEN)), 0); +} + +static void l1_vmx_code_part2(void) +{ + u64 control; + + control = vmreadz(CPU_BASED_VM_EXEC_CONTROL); + control &= ~CPU_BASED_TPR_SHADOW; + GUEST_ASSERT_EQ(vmwrite(CPU_BASED_VM_EXEC_CONTROL, control), 0); + + control = vmreadz(SECONDARY_VM_EXEC_CONTROL); + control &= ~(SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | + SECONDARY_EXEC_APIC_REGISTER_VIRT | + SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); + GUEST_ASSERT_EQ(vmwrite(SECONDARY_VM_EXEC_CONTROL, control), 0); + + GUEST_ASSERT(!vmresume()); + GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_CPUID); + GUEST_ASSERT_EQ(vmwrite(GUEST_RIP, + vmreadz(GUEST_RIP) + vmreadz(VM_EXIT_INSTRUCTION_LEN)), 0); +} + +static void l1_test_x2apic_intercepts(void) +{ + GUEST_ASSERT_EQ(nr_irqs, 0); + + sti_nop(); + + GUEST_ASSERT_EQ(x2apic_read_reg(APIC_TASKPRI), 0xf0); + + x2apic_write_reg(APIC_ICR, APIC_DEST_SELF | APIC_INT_ASSERT | POSTED_INTR_VECTOR); + GUEST_ASSERT_EQ(nr_irqs, 0); + + x2apic_write_reg(APIC_TASKPRI, 0xff); + GUEST_ASSERT_EQ(x2apic_read_reg(APIC_TASKPRI), 0xff); + x2apic_write_reg(APIC_ICR, APIC_DEST_SELF | APIC_INT_ASSERT | POSTED_INTR_WAKEUP_VECTOR); + GUEST_ASSERT_EQ(nr_irqs, 0); + + x2apic_write_reg(APIC_TASKPRI, 0); + GUEST_ASSERT_EQ(x2apic_read_reg(APIC_TASKPRI), 0); + + x2apic_write_reg(APIC_ICR, APIC_DEST_SELF | APIC_INT_ASSERT | POSTED_INTR_NESTED_VECTOR); + GUEST_ASSERT_EQ(nr_irqs, 3); + + nr_irqs = 0; +} + +static void l1_guest_code(void *test_data, void *hv_pages) +{ + x2apic_enable(); + + if (this_cpu_has(X86_FEATURE_SVM)) + l1_svm_code(test_data); + else + l1_vmx_code(test_data, hv_pages); + + GUEST_ASSERT_EQ(x2apic_read_reg(APIC_TASKPRI), 0); + x2apic_write_reg(APIC_TASKPRI, 0xf0); + + l1_test_x2apic_intercepts(); + + if (this_cpu_has(X86_FEATURE_VMX)) { + l1_vmx_code_part2(); + l1_test_x2apic_intercepts(); + } + + GUEST_DONE(); +} + +static void test_x2apic_intercepts(bool with_inhibit_apicv, bool use_evmcs) +{ + gva_t nested_test_data_gva, hv_pages_gva = 0; + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + struct ucall uc; + + inhibit_apicv = with_inhibit_apicv; + + vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); + vm_install_exception_handler(vm, POSTED_INTR_VECTOR, guest_irq_handler); + vm_install_exception_handler(vm, POSTED_INTR_WAKEUP_VECTOR, guest_irq_handler); + vm_install_exception_handler(vm, POSTED_INTR_NESTED_VECTOR, guest_irq_handler); + + sync_global_to_guest(vm, inhibit_apicv); + + if (kvm_cpu_has(X86_FEATURE_SVM)) + vcpu_alloc_svm(vm, &nested_test_data_gva); + else + vcpu_alloc_vmx(vm, &nested_test_data_gva); + + if (use_evmcs) { + vcpu_set_hv_cpuid(vcpu); + vcpu_enable_evmcs(vcpu); + + vcpu_alloc_hyperv_test_pages(vm, &hv_pages_gva); + } + + vcpu_args_set(vcpu, 2, nested_test_data_gva, hv_pages_gva); + + vcpu_run(vcpu); + + TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO); + + switch (get_ucall(vcpu, &uc)) { + case UCALL_DONE: + break; + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + break; + default: + TEST_FAIL("Expected DONE, got unexpected ucall %lu", uc.cmd); + } + + kvm_vm_free(vm); +} + +int main(int argc, char *argv[]) +{ + TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SVM) || kvm_cpu_has(X86_FEATURE_VMX)); + + test_x2apic_intercepts(true, false); + test_x2apic_intercepts(false, false); + + if (kvm_has_cap(KVM_CAP_HYPERV_ENLIGHTENED_VMCS)) { + test_x2apic_intercepts(true, true); + test_x2apic_intercepts(false, true); + } +} diff --git a/tools/testing/selftests/landlock/fs_test.c b/tools/testing/selftests/landlock/fs_test.c index 18dbdb99aeba..6e979cef884d 100644 --- a/tools/testing/selftests/landlock/fs_test.c +++ b/tools/testing/selftests/landlock/fs_test.c @@ -10493,9 +10493,9 @@ FIXTURE_TEARDOWN_PARENT(trace_layout1) } /* - * Verifies that check_rule_fs events include correct field values: domain, dev, - * ino, access_request, and grants. All values are verified against stat() of - * the rule path on a deterministic tmpfs layout. + * Verifies that check_rule_inode events include correct field values: domain, + * dev, ino, access_request, and grants. All values are verified against stat() + * of the rule path on a deterministic tmpfs layout. */ TEST_F(trace_layout1, check_rule_fs_fields) { @@ -10529,7 +10529,7 @@ TEST_F(trace_layout1, check_rule_fs_fields) EXPECT_EQ(1, tracefs_count_matches(buf, REGEX_CHECK_RULE_FS(TRACE_TASK))) { - TH_LOG("Expected 1 check_rule_fs event\n%s", buf); + TH_LOG("Expected 1 check_rule_inode event\n%s", buf); } ASSERT_EQ(0, tracefs_extract_field(buf, REGEX_CHECK_RULE_FS(TRACE_TASK), @@ -10570,8 +10570,8 @@ TEST_F(trace_layout1, check_rule_fs_fields) } /* - * Verifies check_rule_fs behavior with multiple rules. With rules at s1d1 and - * s1d2 (a child of s1d1), accessing s1d2 produces only 1 event because the + * Verifies check_rule_inode behavior with multiple rules. With rules at s1d1 + * and s1d2 (a child of s1d1), accessing s1d2 produces only 1 event because the * pathwalk short-circuits after the first rule fully unmasks the single layer. */ TEST_F(trace_layout1, check_rule_fs_multiple_rules) @@ -10643,14 +10643,14 @@ TEST_F(trace_layout1, check_rule_fs_multiple_rules) ASSERT_NE(NULL, buf); /* - * Only 1 check_rule_fs event: the rule on dir_s1d2 fully unmasked the - * single layer, so the pathwalk short-circuits before reaching the + * Only one check_rule_inode event: the rule on dir_s1d2 fully unmasks + * the single layer, so the pathwalk short-circuits before reaching the * dir_s1d1 rule. */ count = tracefs_count_matches(buf, REGEX_CHECK_RULE_FS(TRACE_TASK)); EXPECT_EQ(1, count) { - TH_LOG("Expected 1 check_rule_fs event, got %d\n%s", count, + TH_LOG("Expected 1 check_rule_inode event, got %d\n%s", count, buf); } @@ -10777,7 +10777,7 @@ TEST_F(trace_layout1, check_rule_fs_optional_access) count = tracefs_count_matches(buf, REGEX_CHECK_RULE_FS(TRACE_TASK)); EXPECT_EQ(1, count) { - TH_LOG("Expected 1 check_rule_fs event, got %d\n%s", count, + TH_LOG("Expected 1 check_rule_inode event, got %d\n%s", count, buf); } @@ -10796,7 +10796,7 @@ TEST_F(trace_layout1, check_rule_fs_optional_access) } /* - * Verifies that check_rule_fs fires for a rule that matches the inode even when + * Verifies that check_rule_inode fires for a rule matching the inode even when * it grants none of the requested rights, so the grants set is empty. Landlock * cannot know a rule ignores the request before reading it, so the event is * still emitted (grants={}), which lets a tracer see that the rule matched. @@ -10884,7 +10884,7 @@ TEST_F(trace_layout1, check_rule_fs_empty_grant) count = tracefs_count_matches(buf, REGEX_CHECK_RULE_FS(TRACE_TASK)); EXPECT_EQ(2, count) { - TH_LOG("Expected 2 check_rule_fs events, got %d\n%s", count, + TH_LOG("Expected 2 check_rule_inode events, got %d\n%s", count, buf); } @@ -10894,7 +10894,7 @@ TEST_F(trace_layout1, check_rule_fs_empty_grant) tracefs_count_matches( buf, TRACE_PREFIX( - TRACE_TASK) "landlock_check_rule_fs: domain=[0-9a-f]\\+ " + TRACE_TASK) "landlock_check_rule_inode: domain=[0-9a-f]\\+ " "access_request=read_dir " "dev=[0-9]\\+:[0-9]\\+ ino=[0-9]\\+ " "grants={}$")) @@ -10908,7 +10908,7 @@ TEST_F(trace_layout1, check_rule_fs_empty_grant) tracefs_count_matches( buf, TRACE_PREFIX( - TRACE_TASK) "landlock_check_rule_fs: domain=[0-9a-f]\\+ " + TRACE_TASK) "landlock_check_rule_inode: domain=[0-9a-f]\\+ " "access_request=read_dir " "dev=[0-9]\\+:[0-9]\\+ ino=[0-9]\\+ " "grants={read_dir}$")) diff --git a/tools/testing/selftests/landlock/net_test.c b/tools/testing/selftests/landlock/net_test.c index a18761e0fd82..16afbfdf06bb 100644 --- a/tools/testing/selftests/landlock/net_test.c +++ b/tools/testing/selftests/landlock/net_test.c @@ -3481,13 +3481,14 @@ TEST_F(trace_net, deny_access_net_bind) } /* - * Anchors the denial fields shared by every deny_access_net event so a field - * test proves more than sport/dport: the denying domain, the same-exec bit, the - * audit-logging verdict, and the blocked access all stay populated. + * Anchors the denial fields shared by every deny_access_net event so a port + * test also proves the denying domain, execution status, logging verdict, and + * exact blocked access. */ static void expect_net_deny_common_fields(struct __test_metadata *const _metadata, - const char *const buf) + const char *const buf, + const char *const expected_blockers) { char field[64]; @@ -3511,18 +3512,21 @@ expect_net_deny_common_fields(struct __test_metadata *const _metadata, ASSERT_EQ(0, tracefs_extract_field(buf, REGEX_DENY_ACCESS_NET(TRACE_TASK), "blockers", field, sizeof(field))); - EXPECT_STRNE("", field); + EXPECT_STREQ(expected_blockers, field); } -/* Connect and field-check tests use a separate fixture without variants. */ +enum trace_net_operation { + TRACE_NET_BIND, + TRACE_NET_SEND, +}; /* clang-format off */ -FIXTURE(trace_net_connect) { +FIXTURE(trace_net_address) { /* clang-format on */ int tracefs_ok; }; -FIXTURE_SETUP(trace_net_connect) +FIXTURE_SETUP(trace_net_address) { int ret; @@ -3547,7 +3551,7 @@ FIXTURE_SETUP(trace_net_connect) clear_cap(_metadata, CAP_SYS_ADMIN); } -FIXTURE_TEARDOWN(trace_net_connect) +FIXTURE_TEARDOWN(trace_net_address) { if (!self->tracefs_ok) return; @@ -3559,160 +3563,183 @@ FIXTURE_TEARDOWN(trace_net_connect) } /* clang-format off */ -FIXTURE_VARIANT(trace_net_connect) { +FIXTURE_VARIANT(trace_net_address) { /* clang-format on */ - /* handled_access_net, also the access allowed on the base port. */ - __u64 handled; - /* Bind the allowed base port before the denied operation. */ - bool bind_base_first; - /* Denied operation on the next port: connect (true) or bind (false). */ - bool deny_connect; + int socket_family; + int socket_type; + enum trace_net_operation operation; + int address_family; + socklen_t addrlen; + __u64 handled_access; + const char *expected_blockers; + bool address_port_zero; + bool expected_address_port; + int expected_port; }; /* clang-format off */ - -/* Denied connect(): sport=0, dport=. */ -FIXTURE_VARIANT_ADD(trace_net_connect, connect_denied) { - .handled = LANDLOCK_ACCESS_NET_CONNECT_TCP, - .bind_base_first = false, - .deny_connect = true, +FIXTURE_VARIANT_ADD(trace_net_address, ipv4_tcp_bind) { + /* clang-format on */ + .socket_family = AF_INET, + .socket_type = SOCK_STREAM, + .operation = TRACE_NET_BIND, + .address_family = AF_INET, + .addrlen = sizeof(struct sockaddr_in), + .handled_access = LANDLOCK_ACCESS_NET_BIND_TCP, + .expected_blockers = "bind_tcp", + .expected_address_port = true, }; -/* Denied bind(): sport=, dport=0. */ -FIXTURE_VARIANT_ADD(trace_net_connect, bind_fields) { - .handled = LANDLOCK_ACCESS_NET_BIND_TCP, - .bind_base_first = false, - .deny_connect = false, +/* Explicit bind(0) has a checked zero port. */ +/* clang-format off */ +FIXTURE_VARIANT_ADD(trace_net_address, ipv4_udp_bind_zero) { + /* clang-format on */ + .socket_family = AF_INET, + .socket_type = SOCK_DGRAM, + .operation = TRACE_NET_BIND, + .address_family = AF_INET, + .addrlen = sizeof(struct sockaddr_in), + .handled_access = LANDLOCK_ACCESS_NET_BIND_UDP, + .expected_blockers = "bind_udp", + .address_port_zero = true, + .expected_port = 0, }; -/* Denied connect() after an allowed bind(): the connect fields (sport=0). */ -FIXTURE_VARIANT_ADD(trace_net_connect, connect_after_bind) { - .handled = LANDLOCK_ACCESS_NET_BIND_TCP | LANDLOCK_ACCESS_NET_CONNECT_TCP, - .bind_base_first = true, - .deny_connect = true, +/* A UDP send can deny its synthetic unspecified bind endpoint. */ +/* clang-format off */ +FIXTURE_VARIANT_ADD(trace_net_address, ipv6_udp_autobind) { + /* clang-format on */ + .socket_family = AF_INET6, + .socket_type = SOCK_DGRAM, + .operation = TRACE_NET_SEND, + .address_family = AF_INET6, + .addrlen = sizeof(struct sockaddr_in6), + .handled_access = LANDLOCK_ACCESS_NET_BIND_UDP, + .expected_blockers = "bind_udp", + .expected_port = 0, }; -/* clang-format on */ +/* A family-only address has no checked port. */ +/* clang-format off */ +FIXTURE_VARIANT_ADD(trace_net_address, ipv6_unspec_udp_send_min) { + /* clang-format on */ + .socket_family = AF_INET6, + .socket_type = SOCK_DGRAM, + .operation = TRACE_NET_SEND, + .address_family = AF_UNSPEC, + .addrlen = sizeof(sa_family_t), + .handled_access = LANDLOCK_ACCESS_NET_CONNECT_SEND_UDP, + .expected_blockers = "connect_send_udp", + .expected_port = -1, +}; -/* - * A denied TCP bind(2) or connect(2) emits one deny_access_net event. The port - * is reported in the field matching the denied operation, in host endianness - * (the UAPI landlock_net_port_attr.port convention): a connect denial reports - * sport=0 dport=, a bind denial reports sport= dport=0, so a - * byte-order or field-swap bug is caught. A prior allowed bind - * (connect_after_bind) does not change the connect denial's fields. - */ -TEST_F(trace_net_connect, deny_access_net) +static void set_trace_net_address(struct sockaddr_storage *const storage, + const int socket_family, + const int address_family, + const unsigned short port) { - pid_t child; - int status; - char *buf; + memset(storage, 0, sizeof(*storage)); + + if (socket_family == AF_INET) { + struct sockaddr_in *const addr4 = (struct sockaddr_in *)storage; + + addr4->sin_family = address_family; + addr4->sin_port = htons(port); + addr4->sin_addr.s_addr = htonl(INADDR_LOOPBACK); + } else { + struct sockaddr_in6 *const addr6 = + (struct sockaddr_in6 *)storage; + + addr6->sin6_family = address_family; + addr6->sin6_port = htons(port); + addr6->sin6_addr = in6addr_loopback; + } +} + +/* Verifies the actionable signed port for representative checked shapes. */ +TEST_F(trace_net_address, deny_access_net) +{ + const char *const event_regex = REGEX_DENY_ACCESS_NET(TRACE_TASK); + const unsigned short address_port = + variant->address_port_zero ? 0 : sock_port_start + 1; + const int expected_port = variant->expected_address_port ? + address_port : + variant->expected_port; + const struct landlock_ruleset_attr ruleset_attr = { + .handled_access_net = variant->handled_access, + }; + struct sockaddr_storage address; char field[64], expected[16]; + char *buf; + int count, ret, ruleset_fd, socket_fd, status; + pid_t child; if (!self->tracefs_ok) SKIP(return, "tracefs not available"); + set_trace_net_address(&address, variant->socket_family, + variant->address_family, address_port); + socket_fd = socket(variant->socket_family, + variant->socket_type | SOCK_CLOEXEC, 0); + ASSERT_LE(0, socket_fd); + ruleset_fd = + landlock_create_ruleset(&ruleset_attr, sizeof(ruleset_attr), 0); + ASSERT_LE(0, ruleset_fd); + ASSERT_EQ(0, tracefs_clear_buf()); + child = fork(); ASSERT_LE(0, child); - if (child == 0) { - struct landlock_ruleset_attr ruleset_attr = { - .handled_access_net = variant->handled, - }; - struct landlock_net_port_attr port_attr = { - .allowed_access = variant->handled, - .port = sock_port_start, - }; - struct sockaddr_in addr = { - .sin_family = AF_INET, - .sin_addr.s_addr = htonl(INADDR_LOOPBACK), - }; - int ruleset_fd, sock_fd, optval = 1, ret; - - ruleset_fd = landlock_create_ruleset(&ruleset_attr, - sizeof(ruleset_attr), 0); - if (ruleset_fd < 0) + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) _exit(1); - if (landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT, - &port_attr, 0)) { - close(ruleset_fd); - _exit(1); - } - prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); - if (landlock_restrict_self(ruleset_fd, 0)) { - close(ruleset_fd); - _exit(1); - } + if (landlock_restrict_self(ruleset_fd, 0)) + _exit(2); close(ruleset_fd); - sock_fd = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0); - if (sock_fd < 0) - _exit(1); - - /* Bind the allowed base port first (succeeds, no event). */ - if (variant->bind_base_first) { - setsockopt(sock_fd, SOL_SOCKET, SO_REUSEADDR, &optval, - sizeof(optval)); - addr.sin_port = htons(sock_port_start); - if (bind(sock_fd, (struct sockaddr *)&addr, - sizeof(addr))) { - close(sock_fd); - _exit(1); - } - } - - /* Denied operation on the next port. */ - addr.sin_port = htons(sock_port_start + 1); - if (variant->deny_connect) - ret = connect(sock_fd, (struct sockaddr *)&addr, - sizeof(addr)); - else - ret = bind(sock_fd, (struct sockaddr *)&addr, - sizeof(addr)); - if (ret == 0) { - close(sock_fd); - _exit(2); - } - if (errno != EACCES) { - close(sock_fd); + switch (variant->operation) { + case TRACE_NET_BIND: + ret = bind(socket_fd, (const struct sockaddr *)&address, + variant->addrlen); + break; + case TRACE_NET_SEND: + ret = sendto(socket_fd, "A", 1, MSG_NOSIGNAL, + (const struct sockaddr *)&address, + variant->addrlen); + break; + default: _exit(3); } - close(sock_fd); + if (ret >= 0 || errno != EACCES) + _exit(4); + close(socket_fd); + _exit(0); } + close(ruleset_fd); + close(socket_fd); ASSERT_EQ(child, waitpid(child, &status, 0)); ASSERT_TRUE(WIFEXITED(status)); - EXPECT_EQ(0, WEXITSTATUS(status)); + ASSERT_EQ(0, WEXITSTATUS(status)); buf = tracefs_read_buf(); ASSERT_NE(NULL, buf); + count = tracefs_count_matches(buf, event_regex); + if (count != 1) + TH_LOG("Expected 1 denial event, got %d\n%s", count, buf); + ASSERT_EQ(1, count); + expect_net_deny_common_fields(_metadata, buf, + variant->expected_blockers); - EXPECT_EQ(1, tracefs_count_matches(buf, - REGEX_DENY_ACCESS_NET(TRACE_TASK))); - - expect_net_deny_common_fields(_metadata, buf); - - /* - * The denied operation's port field carries the port; the other is 0. - */ - snprintf(expected, sizeof(expected), "%llu", - (unsigned long long)(sock_port_start + 1)); - - ASSERT_EQ(0, - tracefs_extract_field(buf, REGEX_DENY_ACCESS_NET(TRACE_TASK), - "sport", field, sizeof(field))); - EXPECT_STREQ(variant->deny_connect ? "0" : expected, field); - - ASSERT_EQ(0, - tracefs_extract_field(buf, REGEX_DENY_ACCESS_NET(TRACE_TASK), - "dport", field, sizeof(field))); - EXPECT_STREQ(variant->deny_connect ? expected : "0", field); + ASSERT_EQ(0, tracefs_extract_field(buf, event_regex, "port", field, + sizeof(field))); + snprintf(expected, sizeof(expected), "%d", expected_port); + EXPECT_STREQ(expected, field); free(buf); } -/* Field verification for the check_rule_net event on an allowed access. */ +/* Field verification for the check_rule_net_port event on an allowed access. */ /* clang-format off */ FIXTURE(trace_net_check_rule) { @@ -3757,10 +3784,11 @@ FIXTURE_TEARDOWN(trace_net_check_rule) /* * Verifies that an allowed bind matching a net-port rule emits exactly one - * landlock_check_rule_net event with the enforcing domain, the requested + * landlock_check_rule_net_port event with the enforcing domain, the requested * access, the checked port (host endianness), and the per-layer grants. The - * whole event is anchored to exact values so a revert of the check_rule_net - * emit (or a byte-order or field-plumbing regression) fails the test. + * whole event is anchored to exact values so removing the check_rule_net_port + * emission or introducing a byte-order or field-plumbing regression fails the + * test. */ TEST_F(trace_net_check_rule, check_rule_net_fields) { @@ -3832,7 +3860,7 @@ TEST_F(trace_net_check_rule, check_rule_net_fields) EXPECT_EQ(1, tracefs_count_matches(buf, REGEX_CHECK_RULE_NET(TRACE_TASK))) { - TH_LOG("Expected 1 check_rule_net event\n%s", buf); + TH_LOG("Expected 1 check_rule_net_port event\n%s", buf); } ASSERT_EQ(0, @@ -3866,11 +3894,4 @@ TEST_F(trace_net_check_rule, check_rule_net_fields) free(buf); } -/* - * IPv6 network trace tests are intentionally elided. IPv6 hook dispatch uses - * the same current_check_access_socket() code path as IPv4, validated by the - * audit tests in this file. The trace events use the same blockers/sport/dport - * fields regardless of address family. - */ - TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/landlock/trace.h b/tools/testing/selftests/landlock/trace.h index ba0c5e92001f..2ec863362173 100644 --- a/tools/testing/selftests/landlock/trace.h +++ b/tools/testing/selftests/landlock/trace.h @@ -27,14 +27,14 @@ TRACEFS_LANDLOCK_DIR "/landlock_create_domain/enable" #define TRACEFS_ENFORCE_DOMAIN_ENABLE \ TRACEFS_LANDLOCK_DIR "/landlock_enforce_domain/enable" -#define TRACEFS_ADD_RULE_FS_ENABLE \ - TRACEFS_LANDLOCK_DIR "/landlock_add_rule_fs/enable" -#define TRACEFS_ADD_RULE_NET_ENABLE \ - TRACEFS_LANDLOCK_DIR "/landlock_add_rule_net/enable" +#define TRACEFS_ADD_RULE_PATH_BENEATH_ENABLE \ + TRACEFS_LANDLOCK_DIR "/landlock_add_rule_path_beneath/enable" +#define TRACEFS_ADD_RULE_NET_PORT_ENABLE \ + TRACEFS_LANDLOCK_DIR "/landlock_add_rule_net_port/enable" #define TRACEFS_CHECK_RULE_FS_ENABLE \ - TRACEFS_LANDLOCK_DIR "/landlock_check_rule_fs/enable" + TRACEFS_LANDLOCK_DIR "/landlock_check_rule_inode/enable" #define TRACEFS_CHECK_RULE_NET_ENABLE \ - TRACEFS_LANDLOCK_DIR "/landlock_check_rule_net/enable" + TRACEFS_LANDLOCK_DIR "/landlock_check_rule_net_port/enable" #define TRACEFS_DENY_ACCESS_FS_ENABLE \ TRACEFS_LANDLOCK_DIR "/landlock_deny_access_fs/enable" #define TRACEFS_DENY_ACCESS_NET_ENABLE \ @@ -79,18 +79,18 @@ */ #define KWORKER_TASK "kworker/[0-9]\\+:[0-9]\\+" -#define REGEX_ADD_RULE_FS(task) \ - TRACE_PREFIX(task) \ - "landlock_add_rule_fs: " \ - "ruleset=[0-9a-f]\\+\\.[0-9]\\+ " \ - "access_rights=[a-z_|]* " \ - "dev=[0-9]\\+:[0-9]\\+ " \ - "ino=[0-9]\\+ " \ +#define REGEX_ADD_RULE_PATH_BENEATH(task) \ + TRACE_PREFIX(task) \ + "landlock_add_rule_path_beneath: " \ + "ruleset=[0-9a-f]\\+\\.[0-9]\\+ " \ + "access_rights=[a-z_|]* " \ + "dev=[0-9]\\+:[0-9]\\+ " \ + "ino=[0-9]\\+ " \ "path=[^ ]\\+$" -#define REGEX_ADD_RULE_NET(task) \ +#define REGEX_ADD_RULE_NET_PORT(task) \ TRACE_PREFIX(task) \ - "landlock_add_rule_net: " \ + "landlock_add_rule_net_port: " \ "ruleset=[0-9a-f]\\+\\.[0-9]\\+ " \ "access_rights=[a-z_|]* " \ "port=[0-9]\\+$" @@ -110,21 +110,21 @@ "parent=[0-9a-f]\\+ " \ "ruleset=[0-9a-f]\\+\\.[0-9]\\+$" -#define REGEX_CHECK_RULE_FS(task) \ - TRACE_PREFIX(task) \ - "landlock_check_rule_fs: " \ - "domain=[0-9a-f]\\+ " \ - "access_request=[a-z_|]* " \ - "dev=[0-9]\\+:[0-9]\\+ " \ - "ino=[0-9]\\+ " \ +#define REGEX_CHECK_RULE_FS(task) \ + TRACE_PREFIX(task) \ + "landlock_check_rule_inode: " \ + "domain=[0-9a-f]\\+ " \ + "access_request=[a-z_|]* " \ + "dev=[0-9]\\+:[0-9]\\+ " \ + "ino=[0-9]\\+ " \ "grants={[a-z_|,]*}$" -#define REGEX_CHECK_RULE_NET(task) \ - TRACE_PREFIX(task) \ - "landlock_check_rule_net: " \ - "domain=[0-9a-f]\\+ " \ - "access_request=[a-z_|]* " \ - "port=[0-9]\\+ " \ +#define REGEX_CHECK_RULE_NET(task) \ + TRACE_PREFIX(task) \ + "landlock_check_rule_net_port: " \ + "domain=[0-9a-f]\\+ " \ + "access_request=[a-z_|]* " \ + "port=[0-9]\\+ " \ "grants={[a-z_|,]*}$" #define REGEX_DENY_ACCESS_FS(task) \ @@ -145,8 +145,7 @@ "same_exec=[01] " \ "logged=[01] " \ "blockers=[a-z_|]* " \ - "sport=[0-9]\\+ " \ - "dport=[0-9]\\+$" + "port=-\\?[0-9]\\+$" #define REGEX_DENY_PTRACE(task) \ TRACE_PREFIX(task) \ diff --git a/tools/testing/selftests/landlock/trace_fs_test.c b/tools/testing/selftests/landlock/trace_fs_test.c index 4543a25c1f55..64014ade3a0e 100644 --- a/tools/testing/selftests/landlock/trace_fs_test.c +++ b/tools/testing/selftests/landlock/trace_fs_test.c @@ -121,7 +121,8 @@ FIXTURE_SETUP(trace_fs) } self->tracefs_ok = 1; - ASSERT_EQ(0, tracefs_enable_event(TRACEFS_ADD_RULE_FS_ENABLE, true)); + ASSERT_EQ(0, tracefs_enable_event(TRACEFS_ADD_RULE_PATH_BENEATH_ENABLE, + true)); ASSERT_EQ(0, tracefs_enable_event(TRACEFS_CHECK_RULE_FS_ENABLE, true)); ASSERT_EQ(0, tracefs_enable_event(TRACEFS_DENY_ACCESS_FS_ENABLE, true)); ASSERT_EQ(0, tracefs_clear()); @@ -134,7 +135,7 @@ FIXTURE_TEARDOWN(trace_fs) return; set_cap(_metadata, CAP_SYS_ADMIN); - tracefs_enable_event(TRACEFS_ADD_RULE_FS_ENABLE, false); + tracefs_enable_event(TRACEFS_ADD_RULE_PATH_BENEATH_ENABLE, false); tracefs_enable_event(TRACEFS_CHECK_RULE_FS_ENABLE, false); tracefs_enable_event(TRACEFS_DENY_ACCESS_FS_ENABLE, false); tracefs_fixture_teardown(); @@ -183,11 +184,11 @@ TEST_F(trace_fs, unsandboxed) } /* - * Verifies that adding a filesystem rule emits a landlock_add_rule_fs trace - * event with the expected path and field values: ruleset ID is non-zero, - * access_rights is non-zero, and path matches. + * Verifies that adding a filesystem rule emits a landlock_add_rule_path_beneath + * event with the expected path and field values: the ruleset ID and + * access_rights are non-zero, and the path matches. */ -TEST_F(trace_fs, add_rule_fs) +TEST_F(trace_fs, add_rule_path_beneath) { struct landlock_ruleset_attr ruleset_attr = { .handled_access_fs = LANDLOCK_ACCESS_FS_READ_FILE | @@ -215,28 +216,30 @@ TEST_F(trace_fs, add_rule_fs) buf = tracefs_read_buf(); ASSERT_NE(NULL, buf); - count = tracefs_count_matches(buf, REGEX_ADD_RULE_FS(TRACE_TASK)); + count = tracefs_count_matches(buf, + REGEX_ADD_RULE_PATH_BENEATH(TRACE_TASK)); EXPECT_EQ(1, count) { - TH_LOG("Expected 1 add_rule_fs event, got %d\n%s", count, buf); + TH_LOG("Expected 1 add_rule_path_beneath event, got %d\n%s", + count, buf); } /* Ruleset ID should be non-zero. */ - ASSERT_EQ(0, tracefs_extract_field(buf, REGEX_ADD_RULE_FS(TRACE_TASK), - "ruleset", field_buf, - sizeof(field_buf))); + ASSERT_EQ(0, tracefs_extract_field( + buf, REGEX_ADD_RULE_PATH_BENEATH(TRACE_TASK), + "ruleset", field_buf, sizeof(field_buf))); EXPECT_STRNE("0", field_buf); /* Access rights should be non-zero. */ - ASSERT_EQ(0, tracefs_extract_field(buf, REGEX_ADD_RULE_FS(TRACE_TASK), - "access_rights", field_buf, - sizeof(field_buf))); + ASSERT_EQ(0, tracefs_extract_field( + buf, REGEX_ADD_RULE_PATH_BENEATH(TRACE_TASK), + "access_rights", field_buf, sizeof(field_buf))); EXPECT_STRNE("", field_buf); /* Path should be /usr. */ - ASSERT_EQ(0, - tracefs_extract_field(buf, REGEX_ADD_RULE_FS(TRACE_TASK), - "path", field_buf, sizeof(field_buf))); + ASSERT_EQ(0, tracefs_extract_field( + buf, REGEX_ADD_RULE_PATH_BENEATH(TRACE_TASK), + "path", field_buf, sizeof(field_buf))); EXPECT_STREQ("/usr", field_buf); free(buf); @@ -246,7 +249,7 @@ TEST_F(trace_fs, add_rule_fs) * Verifies that a path whose escaping exceeds the trace scratch sequence does * not corrupt a sibling symbolic field. */ -TEST_F(trace_fs, add_rule_fs_escaped_path_overflow) +TEST_F(trace_fs, add_rule_path_beneath_escaped_path_overflow) { static const char access_prefix[] = "execute|write_file|read_file|"; static const char access_suffix[] = "|ioctl_dev|resolve_unix"; @@ -277,10 +280,12 @@ TEST_F(trace_fs, add_rule_fs_escaped_path_overflow) buf = tracefs_read_buf(); ASSERT_NE(NULL, buf); - count = tracefs_count_matches(buf, REGEX_ADD_RULE_FS(TRACE_TASK)); + count = tracefs_count_matches(buf, + REGEX_ADD_RULE_PATH_BENEATH(TRACE_TASK)); EXPECT_EQ(1, count) { - TH_LOG("Expected 1 add_rule_fs event, got %d\n%s", count, buf); + TH_LOG("Expected 1 add_rule_path_beneath event, got %d\n%s", + count, buf); } /* @@ -288,9 +293,9 @@ TEST_F(trace_fs, add_rule_fs_escaped_path_overflow) * field also catches scratch-sequence poisoning when the compiler * evaluates the overflowing path first, as GCC currently does. */ - ASSERT_EQ(0, tracefs_extract_field(buf, REGEX_ADD_RULE_FS(TRACE_TASK), - "access_rights", field_buf, - sizeof(field_buf))); + ASSERT_EQ(0, tracefs_extract_field( + buf, REGEX_ADD_RULE_PATH_BENEATH(TRACE_TASK), + "access_rights", field_buf, sizeof(field_buf))); EXPECT_EQ(0, strncmp(field_buf, access_prefix, sizeof(access_prefix) - 1)); EXPECT_EQ(NULL, strstr(field_buf, "|refer|")); @@ -298,7 +303,8 @@ TEST_F(trace_fs, add_rule_fs_escaped_path_overflow) ASSERT_LE(sizeof(access_suffix) - 1, field_len); EXPECT_STREQ(access_suffix, field_buf + field_len - (sizeof(access_suffix) - 1)); - expect_truncated_path(_metadata, buf, REGEX_ADD_RULE_FS(TRACE_TASK)); + expect_truncated_path(_metadata, buf, + REGEX_ADD_RULE_PATH_BENEATH(TRACE_TASK)); free(buf); } @@ -542,7 +548,8 @@ TEST_F(trace_fs, check_rule_nested) */ TEST_F(trace_fs, deny_access_fs_denied) { - char *buf; + const char *const event_regex = REGEX_DENY_ACCESS_FS(TRACE_TASK); + char *buf, blockers[64]; int count; ASSERT_EQ(0, tracefs_clear_buf()); @@ -558,8 +565,77 @@ TEST_F(trace_fs, deny_access_fs_denied) buf = tracefs_read_buf(); ASSERT_NE(NULL, buf); - count = tracefs_count_matches(buf, REGEX_DENY_ACCESS_FS(TRACE_TASK)); - EXPECT_LE(1, count); + count = tracefs_count_matches(buf, event_regex); + EXPECT_EQ(1, count) + { + TH_LOG("Expected 1 access denial, got %d\n%s", count, buf); + } + ASSERT_EQ(0, tracefs_extract_field(buf, event_regex, "blockers", + blockers, sizeof(blockers))); + EXPECT_STREQ("read_dir", blockers); + + free(buf); +} + +/* + * Verifies that a denied mount reports the singleton topology blocker rather + * than an empty access mask. + */ +TEST_F(trace_fs, deny_change_topology) +{ + const char *const event_regex = REGEX_DENY_ACCESS_FS(TRACE_TASK); + const struct landlock_ruleset_attr ruleset_attr = { + .handled_access_fs = LANDLOCK_ACCESS_FS_REFER, + }; + char *buf, blockers[64]; + int count, ruleset_fd, status; + pid_t pid; + + ruleset_fd = + landlock_create_ruleset(&ruleset_attr, sizeof(ruleset_attr), 0); + ASSERT_LE(0, ruleset_fd); + ASSERT_EQ(0, tracefs_clear_buf()); + + /* Ensure that Landlock is the only expected mount denial. */ + set_cap(_metadata, CAP_SYS_ADMIN); + pid = fork(); + ASSERT_LE(0, pid); + if (pid == 0) { + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) { + close(ruleset_fd); + _exit(1); + } + if (landlock_restrict_self(ruleset_fd, 0)) { + close(ruleset_fd); + _exit(2); + } + close(ruleset_fd); + + if (mount(NULL, "/", NULL, MS_PRIVATE | MS_REC, NULL) != -1) + _exit(3); + + if (errno != EPERM) + _exit(4); + + _exit(0); + } + close(ruleset_fd); + clear_cap(_metadata, CAP_SYS_ADMIN); + + ASSERT_EQ(pid, waitpid(pid, &status, 0)); + ASSERT_TRUE(WIFEXITED(status)); + EXPECT_EQ(0, WEXITSTATUS(status)); + + buf = tracefs_read_buf(); + ASSERT_NE(NULL, buf); + count = tracefs_count_matches(buf, event_regex); + EXPECT_EQ(1, count) + { + TH_LOG("Expected 1 topology denial, got %d\n%s", count, buf); + } + ASSERT_EQ(0, tracefs_extract_field(buf, event_regex, "blockers", + blockers, sizeof(blockers))); + EXPECT_STREQ("change_topology", blockers); free(buf); } diff --git a/tools/testing/selftests/landlock/trace_test.c b/tools/testing/selftests/landlock/trace_test.c index afdaf8511b3a..f9b293a9dd56 100644 --- a/tools/testing/selftests/landlock/trace_test.c +++ b/tools/testing/selftests/landlock/trace_test.c @@ -49,8 +49,10 @@ FIXTURE_SETUP(trace) ASSERT_EQ(0, tracefs_enable_event(TRACEFS_CREATE_RULESET_ENABLE, true)); ASSERT_EQ(0, tracefs_enable_event(TRACEFS_CREATE_DOMAIN_ENABLE, true)); ASSERT_EQ(0, tracefs_enable_event(TRACEFS_ENFORCE_DOMAIN_ENABLE, true)); - ASSERT_EQ(0, tracefs_enable_event(TRACEFS_ADD_RULE_FS_ENABLE, true)); - ASSERT_EQ(0, tracefs_enable_event(TRACEFS_ADD_RULE_NET_ENABLE, true)); + ASSERT_EQ(0, tracefs_enable_event(TRACEFS_ADD_RULE_PATH_BENEATH_ENABLE, + true)); + ASSERT_EQ(0, + tracefs_enable_event(TRACEFS_ADD_RULE_NET_PORT_ENABLE, true)); ASSERT_EQ(0, tracefs_enable_event(TRACEFS_CHECK_RULE_FS_ENABLE, true)); ASSERT_EQ(0, tracefs_enable_event(TRACEFS_CHECK_RULE_NET_ENABLE, true)); ASSERT_EQ(0, tracefs_enable_event(TRACEFS_DENY_ACCESS_FS_ENABLE, true)); @@ -72,8 +74,8 @@ FIXTURE_TEARDOWN(trace) tracefs_enable_event(TRACEFS_CREATE_RULESET_ENABLE, false); tracefs_enable_event(TRACEFS_CREATE_DOMAIN_ENABLE, false); tracefs_enable_event(TRACEFS_ENFORCE_DOMAIN_ENABLE, false); - tracefs_enable_event(TRACEFS_ADD_RULE_FS_ENABLE, false); - tracefs_enable_event(TRACEFS_ADD_RULE_NET_ENABLE, false); + tracefs_enable_event(TRACEFS_ADD_RULE_PATH_BENEATH_ENABLE, false); + tracefs_enable_event(TRACEFS_ADD_RULE_NET_PORT_ENABLE, false); tracefs_enable_event(TRACEFS_CHECK_RULE_FS_ENABLE, false); tracefs_enable_event(TRACEFS_CHECK_RULE_NET_ENABLE, false); tracefs_enable_event(TRACEFS_DENY_ACCESS_FS_ENABLE, false); @@ -103,8 +105,10 @@ TEST_F(trace, no_trace_when_disabled) ASSERT_EQ(0, tracefs_enable_event(TRACEFS_CREATE_DOMAIN_ENABLE, false)); ASSERT_EQ(0, tracefs_enable_event(TRACEFS_ENFORCE_DOMAIN_ENABLE, false)); - ASSERT_EQ(0, tracefs_enable_event(TRACEFS_ADD_RULE_FS_ENABLE, false)); - ASSERT_EQ(0, tracefs_enable_event(TRACEFS_ADD_RULE_NET_ENABLE, false)); + ASSERT_EQ(0, tracefs_enable_event(TRACEFS_ADD_RULE_PATH_BENEATH_ENABLE, + false)); + ASSERT_EQ(0, tracefs_enable_event(TRACEFS_ADD_RULE_NET_PORT_ENABLE, + false)); ASSERT_EQ(0, tracefs_enable_event(TRACEFS_CHECK_RULE_FS_ENABLE, false)); ASSERT_EQ(0, tracefs_enable_event(TRACEFS_CHECK_RULE_NET_ENABLE, false)); @@ -265,10 +269,11 @@ TEST_F(trace, ruleset_version) ASSERT_NE(0, !!dot); EXPECT_STREQ("0", dot + 1); - /* Verify 2 add_rule_fs events were emitted. */ - EXPECT_EQ(2, tracefs_count_matches(buf, REGEX_ADD_RULE_FS(TRACE_TASK))) + /* Verify two add_rule_path_beneath events were emitted. */ + EXPECT_EQ(2, tracefs_count_matches( + buf, REGEX_ADD_RULE_PATH_BENEATH(TRACE_TASK))) { - TH_LOG("Expected 2 add_rule_fs events\n%s", buf); + TH_LOG("Expected 2 add_rule_path_beneath events\n%s", buf); } /* @@ -373,7 +378,7 @@ TEST_F(trace, create_domain) tracefs_count_matches(buf, REGEX_CHECK_RULE_FS(TRACE_TASK)); ASSERT_LE(1, check_count) { - TH_LOG("Expected check_rule_fs events\n%s", buf); + TH_LOG("Expected check_rule_inode events\n%s", buf); } EXPECT_EQ(0, tracefs_extract_field(buf, REGEX_CHECK_RULE_FS(TRACE_TASK), @@ -508,9 +513,11 @@ TEST_F(trace, add_rule_invalid_fd) buf = tracefs_read_buf(); ASSERT_NE(NULL, buf); - EXPECT_EQ(0, tracefs_count_matches(buf, REGEX_ADD_RULE_FS(TRACE_TASK))) + EXPECT_EQ(0, tracefs_count_matches( + buf, REGEX_ADD_RULE_PATH_BENEATH(TRACE_TASK))) { - TH_LOG("No add_rule_fs event expected on invalid fd\n%s", buf); + TH_LOG("No add_rule_path_beneath event expected on invalid fd\n%s", + buf); } free(buf); @@ -902,10 +909,10 @@ TEST_F(trace, non_audit_visible_denial_counting) } /* - * Verifies that landlock_add_rule_net emits a trace event with the correct port - * and allowed access mask fields. + * Verifies that landlock_add_rule_net_port emits a trace event with the correct + * port and allowed access mask fields. */ -TEST_F(trace, add_rule_net_fields) +TEST_F(trace, add_rule_net_port_fields) { struct landlock_ruleset_attr ruleset_attr = { .handled_access_net = LANDLOCK_ACCESS_NET_BIND_TCP, @@ -931,9 +938,10 @@ TEST_F(trace, add_rule_net_fields) buf = tracefs_read_buf(); ASSERT_NE(NULL, buf); - EXPECT_EQ(1, tracefs_count_matches(buf, REGEX_ADD_RULE_NET(TRACE_TASK))) + EXPECT_EQ(1, tracefs_count_matches(buf, + REGEX_ADD_RULE_NET_PORT(TRACE_TASK))) { - TH_LOG("Expected 1 add_rule_net event\n%s", buf); + TH_LOG("Expected 1 add_rule_net_port event\n%s", buf); } /* @@ -941,7 +949,8 @@ TEST_F(trace, add_rule_net_fields) * (landlock_net_port_attr.port). On little-endian, htons(8080) is * 36895, so this comparison catches byte-order bugs. */ - EXPECT_EQ(0, tracefs_extract_field(buf, REGEX_ADD_RULE_NET(TRACE_TASK), + EXPECT_EQ(0, tracefs_extract_field(buf, + REGEX_ADD_RULE_NET_PORT(TRACE_TASK), "port", field, sizeof(field))); EXPECT_STREQ("8080", field); /* @@ -950,9 +959,9 @@ TEST_F(trace, add_rule_net_fields) * net access bits are unhandled because the ruleset only handles * BIND_TCP). */ - EXPECT_EQ(0, - tracefs_extract_field(buf, REGEX_ADD_RULE_NET(TRACE_TASK), - "access_rights", field, sizeof(field))); + EXPECT_EQ(0, tracefs_extract_field( + buf, REGEX_ADD_RULE_NET_PORT(TRACE_TASK), + "access_rights", field, sizeof(field))); EXPECT_STREQ("bind_tcp|connect_tcp|bind_udp|connect_send_udp", field); free(buf); diff --git a/tools/testing/selftests/nci/nci_dev.c b/tools/testing/selftests/nci/nci_dev.c index 312f84ee0444..07427fa42888 100644 --- a/tools/testing/selftests/nci/nci_dev.c +++ b/tools/testing/selftests/nci/nci_dev.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -87,6 +88,16 @@ struct msgtemplate { char buf[MAX_MSG_SIZE]; }; +static int join_thread_status(pthread_t thread) +{ + void *thread_ret = NULL; + + if (pthread_join(thread, &thread_ret)) + return -1; + + return (int)(intptr_t)thread_ret; +} + static int create_nl_socket(void) { int fd; @@ -182,7 +193,7 @@ static int get_family_id(int sd, __u32 pid, __u32 *event_group) } ans; struct nlattr *na; int resp_len; - __u16 id; + __u16 id = 0; int len; int rc; @@ -438,13 +449,13 @@ FIXTURE_SETUP(NCI) else rc = pthread_create(&thread_t, NULL, virtual_dev_open, (void *)&self->virtual_nci_fd); - ASSERT_GT(rc, -1); + ASSERT_EQ(rc, 0); rc = send_cmd_with_idx(self->sd, self->fid, self->pid, NFC_CMD_DEV_UP, self->dev_idex); EXPECT_EQ(rc, 0); - pthread_join(thread_t, (void **)&status); + status = join_thread_status(thread_t); ASSERT_EQ(status, 0); self->open_state = true; } @@ -509,12 +520,12 @@ FIXTURE_TEARDOWN(NCI) rc = pthread_create(&thread_t, NULL, virtual_deinit, (void *)&self->virtual_nci_fd); - ASSERT_GT(rc, -1); + ASSERT_EQ(rc, 0); rc = send_cmd_with_idx(self->sd, self->fid, self->pid, NFC_CMD_DEV_DOWN, self->dev_idex); EXPECT_EQ(rc, 0); - pthread_join(thread_t, (void **)&status); + status = join_thread_status(thread_t); ASSERT_EQ(status, 0); } @@ -585,12 +596,11 @@ int start_polling(int dev_idx, int proto, int virtual_fd, int sd, int fid, int p void *nla_start_poll_data[2] = {&dev_idx, &proto}; int nla_start_poll_len[2] = {4, 4}; pthread_t thread_t; - int status; int rc; rc = pthread_create(&thread_t, NULL, virtual_poll_start, (void *)&virtual_fd); - if (rc < 0) + if (rc) return rc; rc = send_cmd_mt_nla(sd, fid, pid, NFC_CMD_START_POLL, 2, nla_start_poll_type, @@ -598,19 +608,17 @@ int start_polling(int dev_idx, int proto, int virtual_fd, int sd, int fid, int p if (rc != 0) return rc; - pthread_join(thread_t, (void **)&status); - return status; + return join_thread_status(thread_t); } int stop_polling(int dev_idx, int virtual_fd, int sd, int fid, int pid) { pthread_t thread_t; - int status; int rc; rc = pthread_create(&thread_t, NULL, virtual_poll_stop, (void *)&virtual_fd); - if (rc < 0) + if (rc) return rc; rc = send_cmd_with_idx(sd, fid, pid, @@ -618,8 +626,7 @@ int stop_polling(int dev_idx, int virtual_fd, int sd, int fid, int pid) if (rc != 0) return rc; - pthread_join(thread_t, (void **)&status); - return status; + return join_thread_status(thread_t); } TEST_F(NCI, start_poll) @@ -830,10 +837,14 @@ int disconnect_tag(int nfc_sock, int virtual_fd) status = pthread_create(&thread_t, NULL, virtual_deactivate_proc, (void *)&virtual_fd); + if (status) + return status; close(nfc_sock); - pthread_join(thread_t, (void **)&status); - return status; + if (status) + return -1; + + return join_thread_status(thread_t); } TEST_F(NCI, t4t_tag_read) @@ -874,13 +885,13 @@ TEST_F(NCI, deinit) else rc = pthread_create(&thread_t, NULL, virtual_deinit, (void *)&self->virtual_nci_fd); - ASSERT_GT(rc, -1); + ASSERT_EQ(rc, 0); rc = send_cmd_with_idx(self->sd, self->fid, self->pid, NFC_CMD_DEV_DOWN, self->dev_idex); EXPECT_EQ(rc, 0); - pthread_join(thread_t, (void **)&status); + status = join_thread_status(thread_t); self->open_state = 0; ASSERT_EQ(status, 0); diff --git a/tools/testing/selftests/net/.gitignore b/tools/testing/selftests/net/.gitignore index c9f46031ac73..dacd36ed8455 100644 --- a/tools/testing/selftests/net/.gitignore +++ b/tools/testing/selftests/net/.gitignore @@ -55,6 +55,7 @@ tools tun txring_overwrite txtimestamp +udp_splice_checksum udpgso udpgso_bench_rx udpgso_bench_tx diff --git a/tools/testing/selftests/net/Makefile b/tools/testing/selftests/net/Makefile index 3ee3378f8b26..590b33e16d9f 100644 --- a/tools/testing/selftests/net/Makefile +++ b/tools/testing/selftests/net/Makefile @@ -199,6 +199,7 @@ TEST_GEN_PROGS := \ tap \ tcp_port_share \ tls \ + udp_splice_checksum \ # end of TEST_GEN_PROGS TEST_FILES := \ diff --git a/tools/testing/selftests/net/config b/tools/testing/selftests/net/config index 30d5fcb09a83..737e7e6327b3 100644 --- a/tools/testing/selftests/net/config +++ b/tools/testing/selftests/net/config @@ -118,10 +118,10 @@ CONFIG_NFT_NAT=m CONFIG_NUMA=y CONFIG_OPENVSWITCH=m CONFIG_PAGE_POOL_STATS=y -CONFIG_SYSCTL=y CONFIG_PSAMPLE=m CONFIG_RPS=y CONFIG_SYN_COOKIES=y +CONFIG_SYSCTL=y CONFIG_SYSFS=y CONFIG_TAP=m CONFIG_TCP_CONG_DCTCP=y diff --git a/tools/testing/selftests/net/nl_nlctrl.py b/tools/testing/selftests/net/nl_nlctrl.py index fe1f66dc9435..237b3d273260 100755 --- a/tools/testing/selftests/net/nl_nlctrl.py +++ b/tools/testing/selftests/net/nl_nlctrl.py @@ -9,40 +9,86 @@ from lib.py import ksft_run, ksft_exit from lib.py import ksft_eq, ksft_ge, ksft_true, ksft_in, ksft_not_in from lib.py import NetdevFamily, EthtoolFamily, NlctrlFamily +# Families we can expect to always be around, and which between them +# cover ops with a do, with a dump, and with both. +FAMILIES = ('nlctrl', 'netdev') -def getfamily_do(ctrl) -> None: - """Query a single family by name and validate its ops.""" - fam = ctrl.getfamily({'family-name': 'netdev'}) - ksft_eq(fam['family-name'], 'netdev') + +def _get_ops(ctrl, name): + """Get the ops of a family, keyed by command id.""" + fam = ctrl.getfamily({'family-name': name}) + ksft_eq(fam['family-name'], name) ksft_true(fam['family-id'] > 0) # The format of ops is quite odd, [{$idx: {"id"...}}, {$idx: {"id"...}}] # Discard the indices and re-key by command id. ops_by_id = {v['id']: v for op in fam['ops'] for v in op.values()} - ksft_eq(len(ops_by_id), len(fam['ops'])) + ksft_eq(len(ops_by_id), len(fam['ops']), + comment=f"{name} lists a command twice") + return ops_by_id - # All ops should have a policy (either do or dump has one) - for op in ops_by_id.values(): - ksft_in('cmd-cap-haspol', op['flags'], - comment=f"op {op['id']} missing haspol") + +def _get_policy_map(ctrl, req): + """ + The policy map in the Netlink replies looks like this: + + [{'family-id': 16, 'op-policy': {'do': 0, 'dump': 0, 'op-id': 3}}, + {'family-id': 16, 'op-policy': {'dump': 1, 'op-id': 4}}, ...] + + Return the mapping: + + {3:{'do','dump'}, 4:{'dump'}} + + The policy itself is discarded here, only return which command has policy. + """ + pol_map = {} + for msg in ctrl.getpolicy(req, dump=True): + if 'op-policy' not in msg: + continue + modes = dict(msg['op-policy']) + cmd = modes.pop('op-id') + ksft_not_in(cmd, pol_map, comment=f"command {cmd} reported twice") + pol_map[cmd] = set(modes.keys()) + return pol_map + + +def getfamily_do(ctrl) -> None: + """Query single families by name and validate their ops.""" + ops = {name: _get_ops(ctrl, name) for name in FAMILIES} + + for name, ops_by_id in ops.items(): + for op in ops_by_id.values(): + # All ops in nlctrl and netdev have a policy + ksft_in('cmd-cap-haspol', op['flags'], + comment=f"{name} op {op['id']} missing haspol") + ksft_true(op['flags'] & {'cmd-cap-do', 'cmd-cap-dump'}, + comment=f"{name} op {op['id']} has no handler") + + # nlctrl getfamily (id 3) does both, getpolicy (id 10) is dump-only + ksft_in('cmd-cap-do', ops['nlctrl'][3]['flags']) + ksft_in('cmd-cap-dump', ops['nlctrl'][3]['flags']) + ksft_not_in('cmd-cap-do', ops['nlctrl'][10]['flags']) + ksft_in('cmd-cap-dump', ops['nlctrl'][10]['flags']) + + netdev = ops['netdev'] # dev-get (id 1) should support both do and dump - ksft_in('cmd-cap-do', ops_by_id[1]['flags']) - ksft_in('cmd-cap-dump', ops_by_id[1]['flags']) + ksft_in('cmd-cap-do', netdev[1]['flags']) + ksft_in('cmd-cap-dump', netdev[1]['flags']) # qstats-get (id 12) is dump-only - ksft_not_in('cmd-cap-do', ops_by_id[12]['flags']) - ksft_in('cmd-cap-dump', ops_by_id[12]['flags']) + ksft_not_in('cmd-cap-do', netdev[12]['flags']) + ksft_in('cmd-cap-dump', netdev[12]['flags']) # napi-set (id 14) is do-only and requires admin - ksft_in('cmd-cap-do', ops_by_id[14]['flags']) - ksft_not_in('cmd-cap-dump', ops_by_id[14]['flags']) - ksft_in('admin-perm', ops_by_id[14]['flags']) + ksft_in('cmd-cap-do', netdev[14]['flags']) + ksft_not_in('cmd-cap-dump', netdev[14]['flags']) + ksft_in('admin-perm', netdev[14]['flags']) # Notification-only commands (dev-add/del/change-ntf etc.) must # not appear in the ops list since they have no do/dump handlers. for ntf_id in [2, 3, 4, 6, 7, 8]: - ksft_not_in(ntf_id, ops_by_id, + ksft_not_in(ntf_id, netdev, comment=f"ntf-only cmd {ntf_id} should not be in ops") @@ -103,6 +149,41 @@ def getpolicy_dump(_ctrl) -> None: comment="linkinfo-set should not have a dump policy") +def getpolicy_op_map(ctrl) -> None: + """Check the op-to-policy map consistency. Each op with 'haspol' flag + has to have a policy. The policy back-references must name only + real ops that exist, have given modes (do vs dump) and have 'haspol'. + """ + for name in FAMILIES: + ops_by_id = _get_ops(ctrl, name) + haspol = {cmd for cmd, op in ops_by_id.items() + if 'cmd-cap-haspol' in op['flags']} + + pol_map = _get_policy_map(ctrl, {'family-name': name}) + ksft_eq(set(pol_map), haspol, + comment=f"{name} policy map does not match the op list") + + # Walk the op list rather than the map, the map may be missing + # the very op we are after. Asking for a command the family does + # not have is an error, so it must not come from the map either. + for cmd in sorted(haspol): + modes = pol_map.get(cmd, set()) + + # The kernel only reports a mode the op actually has. + if 'do' in modes: + ksft_in('cmd-cap-do', ops_by_id[cmd]['flags'], + comment=f"{name} cmd {cmd} has no do") + if 'dump' in modes: + ksft_in('cmd-cap-dump', ops_by_id[cmd]['flags'], + comment=f"{name} cmd {cmd} has no dump") + + # Asking for one op builds the map in a different place in + # the kernel, it has to report what the full dump did. + single = _get_policy_map(ctrl, {'family-name': name, 'op': cmd}) + ksft_eq(single, {cmd: modes}, + comment=f"{name} cmd {cmd} policy differs from the dump") + + def getpolicy_by_op(_ctrl) -> None: """Query policy for specific ops, check attr names are resolved.""" ndev = NetdevFamily() @@ -122,6 +203,7 @@ def main() -> None: ksft_run([getfamily_do, getfamily_dump, getpolicy_dump, + getpolicy_op_map, getpolicy_by_op], args=(ctrl, )) ksft_exit() diff --git a/tools/testing/selftests/net/ovpn/common.sh b/tools/testing/selftests/net/ovpn/common.sh index 2d844eb3aa6e..5e9c81e885e6 100644 --- a/tools/testing/selftests/net/ovpn/common.sh +++ b/tools/testing/selftests/net/ovpn/common.sh @@ -136,6 +136,19 @@ ovpn_create_ns() { ip netns add "ovpn_peer${1}" } +ovpn_peer_vpn_addr() { + local peer="$1" + local file + + if [ "${OVPN_PROTO}" == "UDP" ]; then + file="${OVPN_UDP_PEERS_FILE}" + else + file="${OVPN_TCP_PEERS_FILE}" + fi + + awk -v peer="${peer}" '$1 == peer {print $NF; exit}' "${file}" +} + ovpn_setup_ns() { local peer="ovpn_peer${1}" local server_ns="ovpn_peer0" diff --git a/tools/testing/selftests/net/ovpn/ovpn-cli.c b/tools/testing/selftests/net/ovpn/ovpn-cli.c index f4effa7580c0..3b612a8a18fe 100644 --- a/tools/testing/selftests/net/ovpn/ovpn-cli.c +++ b/tools/testing/selftests/net/ovpn/ovpn-cli.c @@ -650,6 +650,26 @@ static int ovpn_connect(struct ovpn_ctx *ovpn) return ret; } +static int ovpn_nl_put_vpn_addr(struct nl_msg *msg, + const struct ovpn_ctx *ovpn) +{ + if (!ovpn->peer_ip_set) + return 0; + + switch (ovpn->peer_ip.in4.sin_family) { + case AF_INET: + return nla_put_u32(msg, OVPN_A_PEER_VPN_IPV4, + ovpn->peer_ip.in4.sin_addr.s_addr); + case AF_INET6: + return nla_put(msg, OVPN_A_PEER_VPN_IPV6, + sizeof(struct in6_addr), + &ovpn->peer_ip.in6.sin6_addr); + default: + fprintf(stderr, "Invalid family for peer address\n"); + return -EAFNOSUPPORT; + } +} + static int ovpn_new_peer(struct ovpn_ctx *ovpn, bool is_tcp) { struct nlattr *attr; @@ -691,22 +711,9 @@ static int ovpn_new_peer(struct ovpn_ctx *ovpn, bool is_tcp) } } - if (ovpn->peer_ip_set) { - switch (ovpn->peer_ip.in4.sin_family) { - case AF_INET: - NLA_PUT_U32(ctx->nl_msg, OVPN_A_PEER_VPN_IPV4, - ovpn->peer_ip.in4.sin_addr.s_addr); - break; - case AF_INET6: - NLA_PUT(ctx->nl_msg, OVPN_A_PEER_VPN_IPV6, - sizeof(struct in6_addr), - &ovpn->peer_ip.in6.sin6_addr); - break; - default: - fprintf(stderr, "Invalid family for peer address\n"); - goto nla_put_failure; - } - } + ret = ovpn_nl_put_vpn_addr(ctx->nl_msg, ovpn); + if (ret) + goto nla_put_failure; nla_nest_end(ctx->nl_msg, attr); @@ -732,6 +739,10 @@ static int ovpn_set_peer(struct ovpn_ctx *ovpn) ovpn->keepalive_interval); NLA_PUT_U32(ctx->nl_msg, OVPN_A_PEER_KEEPALIVE_TIMEOUT, ovpn->keepalive_timeout); + + ret = ovpn_nl_put_vpn_addr(ctx->nl_msg, ovpn); + if (ret) + goto nla_put_failure; nla_nest_end(ctx->nl_msg, attr); ret = ovpn_nl_msg_send(ctx, NULL); @@ -1730,13 +1741,14 @@ static void usage(const char *cmd) fprintf(stderr, "\tmark: socket FW mark value\n"); fprintf(stderr, - "* set_peer : set peer attributes\n"); + "* set_peer [vpnaddr]: set peer attributes\n"); fprintf(stderr, "\tiface: ovpn interface name\n"); fprintf(stderr, "\tpeer_id: peer ID of the peer to modify\n"); fprintf(stderr, "\tkeepalive_interval: interval for sending ping messages\n"); fprintf(stderr, "\tkeepalive_timeout: time after which a peer is timed out\n"); + fprintf(stderr, "\tvpnaddr: peer VPN IP\n"); fprintf(stderr, "* del_peer : delete peer\n"); fprintf(stderr, "\tiface: ovpn interface name\n"); @@ -2090,6 +2102,8 @@ static int ovpn_run_cmd(struct ovpn_ctx *ovpn) return ret; ret = ovpn_new_peer(ovpn, false); + if (ret < 0) + return ret; ovpn_waitbg(); break; case CMD_NEW_MULTI_PEER: @@ -2331,6 +2345,12 @@ static int ovpn_parse_cmd_args(struct ovpn_ctx *ovpn, int argc, char *argv[]) "keepalive interval value out of range\n"); return -1; } + + if (argc > 6) { + ret = ovpn_parse_remote(ovpn, NULL, NULL, argv[6]); + if (ret < 0) + return -1; + } break; case CMD_DEL_PEER: if (argc < 4) diff --git a/tools/testing/selftests/net/ovpn/test.sh b/tools/testing/selftests/net/ovpn/test.sh index 9b5610837032..392109d5e14e 100755 --- a/tools/testing/selftests/net/ovpn/test.sh +++ b/tools/testing/selftests/net/ovpn/test.sh @@ -56,6 +56,76 @@ ovpn_prepare_network() { done } +ovpn_new_test_peer() { + local peer_id="$1" + + shift + ip netns exec ovpn_peer0 "${OVPN_CLI}" new_peer tun0 \ + "${peer_id}" none 65000 10.10.1.2 1 "$@" +} + +ovpn_set_peer_vpn_addr() { + ip netns exec ovpn_peer0 "${OVPN_CLI}" set_peer tun0 \ + "$1" 60 120 "$2" +} + +ovpn_run_vpn_addr_validation() { + local addr + local peer1_addr4 + local test_peer_id=$((OVPN_NUM_PEERS + 1)) + local test_peer_addr6="2001:db8::2" + # Do not include 0.0.0.0 or :: here. They are invalid on creation, but + # clear one address family on update and are valid if the other remains. + local -a invalid_addrs=( + "127.0.0.1" + "224.0.0.1" + "255.255.255.255" + "::1" + "::192.0.2.1" + "::ffff:192.0.2.1" + "ff02::1" + ) + + peer1_addr4=$(ovpn_peer_vpn_addr 1) + + ovpn_cmd_fail "reject peer without VPN address" \ + ovpn_new_test_peer "${test_peer_id}" + + for addr in "0.0.0.0" "::" "${invalid_addrs[@]}"; do + ovpn_cmd_fail "reject new peer VPN address ${addr}" \ + ovpn_new_test_peer "${test_peer_id}" "${addr}" + done + + ovpn_cmd_fail "reject duplicate IPv4 address on peer creation" \ + ovpn_new_test_peer "${test_peer_id}" "${peer1_addr4}" + ovpn_cmd_fail "reject clearing the last peer VPN address" \ + ovpn_set_peer_vpn_addr 1 0.0.0.0 + + for addr in "${invalid_addrs[@]}"; do + ovpn_cmd_fail "reject updated peer VPN address ${addr}" \ + ovpn_set_peer_vpn_addr 1 "${addr}" + done + + ovpn_cmd_fail "reject duplicate IPv4 address on peer update" \ + ovpn_set_peer_vpn_addr 2 "${peer1_addr4}" + + ovpn_cmd_ok "add peer IPv6 address" \ + ovpn_set_peer_vpn_addr 1 "${test_peer_addr6}" + ovpn_cmd_fail "reject duplicate IPv6 address on peer creation" \ + ovpn_new_test_peer "${test_peer_id}" "${test_peer_addr6}" + ovpn_cmd_fail "reject duplicate IPv6 address on peer update" \ + ovpn_set_peer_vpn_addr 2 "${test_peer_addr6}" + + ovpn_cmd_ok "clear peer IPv4 address" \ + ovpn_set_peer_vpn_addr 1 0.0.0.0 + ovpn_cmd_fail "reject clearing the remaining peer IPv6 address" \ + ovpn_set_peer_vpn_addr 1 :: + ovpn_cmd_ok "restore peer IPv4 address" \ + ovpn_set_peer_vpn_addr 1 "${peer1_addr4}" + ovpn_cmd_ok "clear peer IPv6 address" \ + ovpn_set_peer_vpn_addr 1 :: +} + ovpn_run_basic_traffic() { local p local header1 @@ -293,15 +363,16 @@ trap ovpn_stage_err ERR ktap_print_header if [ "${OVPN_FLOAT}" == "1" ]; then - ktap_set_plan 13 + ktap_set_plan 14 else - ktap_set_plan 12 + ktap_set_plan 13 fi ovpn_cleanup modprobe -q ovpn || true ovpn_run_stage "setup network topology" ovpn_prepare_network +ovpn_run_stage "validate peer VPN addresses" ovpn_run_vpn_addr_validation ovpn_run_stage "run baseline data traffic" ovpn_run_basic_traffic ovpn_run_stage "run LAN traffic behind peer1" ovpn_run_lan_traffic [ "${OVPN_FLOAT}" == "1" ] && ovpn_run_stage "run floating peer checks" \ diff --git a/tools/testing/selftests/net/packetdrill/config b/tools/testing/selftests/net/packetdrill/config index 83dde525c53c..b7df8bf30920 100644 --- a/tools/testing/selftests/net/packetdrill/config +++ b/tools/testing/selftests/net/packetdrill/config @@ -4,8 +4,8 @@ CONFIG_IPV6=y CONFIG_NET_NS=y CONFIG_NET_SCH_FIFO=y CONFIG_NET_SCH_FQ=y -CONFIG_SYSCTL=y CONFIG_SYN_COOKIES=y +CONFIG_SYSCTL=y CONFIG_TCP_CONG_CUBIC=y CONFIG_TCP_MD5SIG=y CONFIG_TUN=y diff --git a/tools/testing/selftests/net/packetdrill/tcp_old_ack_ts.pkt b/tools/testing/selftests/net/packetdrill/tcp_old_ack_ts.pkt new file mode 100644 index 000000000000..f072e5e4c449 --- /dev/null +++ b/tools/testing/selftests/net/packetdrill/tcp_old_ack_ts.pkt @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 +// An out-of-order segment advances SND.UNA. The gap filler still carries +// its original, older ACK. The reply must echo the gap filler's timestamp. +--tcp_ts_tick_usecs=1000 + +`./defaults.sh` + +0.000 socket(..., SOCK_STREAM, IPPROTO_TCP) = 3 ++0 setsockopt(3, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0 ++0 bind(3, ..., ...) = 0 ++0 listen(3, 1) = 0 +0.100 < S 0:0(0) win 20000 ++0 > S. 0:0(0) ack 1 +0.200 < . 1:1(0) ack 1 win 20000 ++0 accept(3, ..., ...) = 4 +0.300 write(4, ..., 17) = 17 ++0 > P. 1:18(17) ack 1 +0.310 < P. 18:35(17) ack 18 win 20000 ++0 > . 18:18(0) ack 1 +0.320 < P. 1:18(17) ack 1 win 20000 ++0 > . 18:18(0) ack 35 ++0 read(4, ..., 34) = 34 diff --git a/tools/testing/selftests/net/udp_splice_checksum.c b/tools/testing/selftests/net/udp_splice_checksum.c new file mode 100644 index 000000000000..dc61444e6481 --- /dev/null +++ b/tools/testing/selftests/net/udp_splice_checksum.c @@ -0,0 +1,378 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Exercise UDP software checksums over multiple spliced pipe fragments. + * All traffic stays on loopback; no privileges or external services are needed. + * Exit status: 0 = pass, 1 = failure, 4 = all cases skipped. + */ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kselftest.h" + +#define TIMEOUT_MS 200 + +struct test_case { + const char *name; + const char *body; + const char *prefix; + size_t fragments[3]; + bool ordinary; + bool page_tail; +}; + +static const struct test_case cases[] = { + { "ordinary-control", "abcDEFGH", "", { 3, 5 }, true, false }, + { "uniform-odd-fragments", "BBBBBBBB", "", { 3, 5 }, false, false }, + { "even-fragments", "abcdEFGH", "", { 4, 4 }, false, false }, + { "odd-fragments", "abcDEFGH", "", { 3, 5 }, false, false }, + { "odd-prefix-odd-fragments", "abcDEFGH", "P", { 3, 5 }, false, false }, + { "even-prefix-odd-fragments", "abcDEFGH", "PQ", { 3, 5 }, false, false }, + { "page-tail-odd-fragments", "abcDEFGH", "", { 3, 5 }, false, true }, + { "three-fragments", "abcDEFGhijkl", "", { 3, 4, 5 }, false, false }, + { "odd-prefix-three-fragments", + "abcDEFGhijkl", + "P", + { 3, 4, 5 }, + false, + false }, +}; + +static const char *const families[] = { "IPv4", "IPv6", "IPv4-mapped" }; + +static int error(const char *what) +{ + ksft_perror(what); + return 1; +} + +static bool unavailable(void) +{ + return errno == EAFNOSUPPORT || errno == EPROTONOSUPPORT || + errno == EADDRNOTAVAIL; +} + +static long long now_ms(void) +{ + struct timespec ts; + + if (clock_gettime(CLOCK_MONOTONIC, &ts)) + ksft_exit_fail_perror("clock_gettime"); + return (long long)ts.tv_sec * 1000 + ts.tv_nsec / 1000000; +} + +static int receive_one(int fd, const char *expected, size_t length) +{ + struct pollfd pfd = { .fd = fd, .events = POLLIN }; + long long deadline = now_ms() + TIMEOUT_MS; + int remaining, ret; + char received[64]; + ssize_t count; + + for (;;) { + remaining = deadline - now_ms(); + if (remaining < 0) + remaining = 0; + ret = poll(&pfd, 1, remaining); + if (ret < 0) { + if (errno == EINTR) + continue; + return error("poll"); + } + if (!ret) { + ksft_print_msg("no datagram within %d ms after send\n", + TIMEOUT_MS); + return 1; + } + count = recv(fd, received, sizeof(received), + MSG_DONTWAIT | MSG_TRUNC); + if (count < 0) { + if (errno == EINTR || errno == EAGAIN) + continue; + return error("recv"); + } + break; + } + if (count != (ssize_t)length || memcmp(received, expected, length)) { + ksft_print_msg("received %zd bytes; expected %zu matching\n", + count, length); + return 1; + } + count = recv(fd, received, sizeof(received), MSG_DONTWAIT | MSG_TRUNC); + if (count >= 0) { + ksft_print_msg("unexpected extra datagram (%zd bytes)\n", + count); + return 1; + } + if (errno != EAGAIN) + return error("extra-datagram check"); + return 0; +} + +/* Separate source pipes prevent adjacent small writes merging into one buffer. */ +static int append_fragment(int dest, const char *data, size_t length) +{ + int source[2], ret = 1; + + if (pipe2(source, O_NONBLOCK | O_CLOEXEC)) + return error("source pipe"); + if (write(source[1], data, length) != (ssize_t)length) { + error("source write"); + goto out; + } + if (splice(source[0], NULL, dest, NULL, length, SPLICE_F_NONBLOCK) != + (ssize_t)length) { + error("pipe-to-pipe splice"); + goto out; + } + ret = 0; +out: + close(source[0]); + close(source[1]); + return ret; +} + +static int pipe_capacity(int fd, unsigned int slots, const char **skip_reason) +{ + long required = slots * sysconf(_SC_PAGESIZE); + int capacity = fcntl(fd, F_GETPIPE_SZ); + + if (capacity < 0) + return error("query pipe capacity"); + if (capacity >= required || + fcntl(fd, F_SETPIPE_SZ, required) >= required) + return 0; + ksft_print_msg("cannot reserve %u pipe slots: %s\n", slots, + strerror(errno)); + *skip_reason = "required pipe capacity unavailable"; + return KSFT_SKIP; +} + +/* Leave three bytes at a page's end, followed by five bytes on the next page. */ +static int append_page_tail(int dest, const char *body, + const char **skip_reason) +{ + long page = sysconf(_SC_PAGESIZE); + int source[2], ret = 1; + char *data; + + if (page < 8) { + ksft_print_msg("invalid page size %ld\n", page); + return 1; + } + data = malloc(page + 5); + if (!data) + return error("malloc"); + memset(data, 'x', page - 3); + memcpy(data + page - 3, body, 8); + if (pipe2(source, O_NONBLOCK | O_CLOEXEC)) { + error("page-tail source pipe"); + goto free_data; + } + ret = pipe_capacity(source[1], 2, skip_reason); + if (ret) + goto out; + ret = 1; + if (write(source[1], data, page + 5) != page + 5) { + error("page-tail write"); + goto out; + } + if (read(source[0], data, page - 3) != page - 3) { + error("page-tail discard"); + goto out; + } + if (splice(source[0], NULL, dest, NULL, 8, SPLICE_F_NONBLOCK) != 8) { + error("page-tail pipe-to-pipe splice"); + goto out; + } + ret = 0; +out: + close(source[0]); + close(source[1]); +free_data: + free(data); + return ret; +} + +static int run_case(unsigned int family, bool connected, + const struct test_case *test, const char **skip_reason) +{ + size_t body_length = strlen(test->body), offset = 0; + int tx_family = family == 0 ? AF_INET : AF_INET6; + int rx_family = family == 1 ? AF_INET6 : AF_INET; + int rx = -1, tx = -1, pipefd[2] = { -1, -1 }; + size_t prefix_length = strlen(test->prefix); + struct sockaddr_storage address = { 0 }; + socklen_t address_length; + struct sockaddr_in6 *v6; + struct sockaddr_in *v4; + int ret = 1, zero = 0; + char expected[64]; + unsigned int i; + ssize_t count; + + v4 = (struct sockaddr_in *)&address; + v6 = (struct sockaddr_in6 *)&address; + *skip_reason = "address family unavailable"; + memcpy(expected, test->prefix, prefix_length); + memcpy(expected + prefix_length, test->body, body_length); + rx = socket(rx_family, SOCK_DGRAM | SOCK_CLOEXEC, 0); + if (rx < 0) { + ret = unavailable() ? KSFT_SKIP : error("receive socket"); + goto out; + } + if (rx_family == AF_INET) { + v4->sin_family = AF_INET; + v4->sin_addr.s_addr = htonl(INADDR_LOOPBACK); + address_length = sizeof(*v4); + } else { + v6->sin6_family = AF_INET6; + v6->sin6_addr = in6addr_loopback; + address_length = sizeof(*v6); + } + if (bind(rx, (struct sockaddr *)&address, address_length)) { + ret = unavailable() ? KSFT_SKIP : error("bind loopback"); + goto out; + } + if (getsockname(rx, (struct sockaddr *)&address, &address_length)) { + error("getsockname"); + goto out; + } + if (family == 2) { + unsigned short port = v4->sin_port; + + memset(&address, 0, sizeof(address)); + v6->sin6_family = AF_INET6; + v6->sin6_port = port; + inet_pton(AF_INET6, "::ffff:127.0.0.1", &v6->sin6_addr); + address_length = sizeof(*v6); + } + tx = socket(tx_family, SOCK_DGRAM | SOCK_CLOEXEC, 0); + if (tx < 0) { + ret = unavailable() ? KSFT_SKIP : error("send socket"); + goto out; + } + if (family == 2 && + setsockopt(tx, IPPROTO_IPV6, IPV6_V6ONLY, &zero, sizeof(zero))) { + error("disable IPV6_V6ONLY"); + goto out; + } + if (connected && + connect(tx, (struct sockaddr *)&address, address_length)) { + error("connect"); + goto out; + } + if (test->ordinary) { + count = sendto(tx, expected, prefix_length + body_length, 0, + (struct sockaddr *)&address, address_length); + if (count != (ssize_t)(prefix_length + body_length)) { + error("ordinary sendto"); + goto out; + } + } else { + if (pipe2(pipefd, O_NONBLOCK | O_CLOEXEC)) { + error("assembled pipe"); + goto out; + } + ret = pipe_capacity(pipefd[1], test->fragments[2] ? 3 : 2, + skip_reason); + if (ret) + goto out; + ret = 1; + if (test->page_tail) { + ret = append_page_tail(pipefd[1], test->body, + skip_reason); + if (ret) + goto out; + ret = 1; + } else { + for (i = 0; i < ARRAY_SIZE(test->fragments); i++) { + if (!test->fragments[i]) + break; + if (append_fragment(pipefd[1], + test->body + offset, + test->fragments[i])) + goto out; + offset += test->fragments[i]; + } + } + /* Even an empty prefix establishes the destination and UDP cork. */ + count = sendto(tx, test->prefix, prefix_length, MSG_MORE, + (struct sockaddr *)&address, address_length); + if (count != (ssize_t)prefix_length) { + error("prime sendto(MSG_MORE)"); + goto out; + } + count = splice(pipefd[0], NULL, tx, NULL, body_length, + SPLICE_F_NONBLOCK | SPLICE_F_MORE); + if (count != (ssize_t)body_length) { + ksft_print_msg("socket splice: %zd, expected %zu: %s\n", + count, body_length, + count < 0 ? strerror(errno) : + "short transfer"); + goto out; + } + if (sendto(tx, "", 0, 0, (struct sockaddr *)&address, + address_length)) { + error("commit sendto"); + goto out; + } + } + ret = receive_one(rx, expected, prefix_length + body_length); +out: + if (pipefd[0] >= 0) { + close(pipefd[0]); + close(pipefd[1]); + } + if (tx >= 0) + close(tx); + if (rx >= 0) + close(rx); + return ret; +} + +int main(void) +{ + unsigned int family, connected, i; + const char *connection, *skip; + int ret; + + ksft_print_header(); + ksft_set_plan(ARRAY_SIZE(families) * 2 * ARRAY_SIZE(cases)); + for (family = 0; family < ARRAY_SIZE(families); family++) { + for (connected = 0; connected < 2; connected++) { + connection = connected ? "connected" : "unconnected"; + for (i = 0; i < ARRAY_SIZE(cases); i++) { + ret = run_case(family, connected, &cases[i], + &skip); + if (ret == KSFT_SKIP) + ksft_test_result_skip("%s %s %s: %s\n", + families[family], + connection, + cases[i].name, + skip); + else + ksft_test_result(!ret, "%s %s %s\n", + families[family], + connection, + cases[i].name); + } + } + } + if (!ksft_get_pass_cnt() && !ksft_get_fail_cnt()) { + /* ksft_exit_skip() would add a result beyond the plan. */ + ksft_print_cnts(); + return KSFT_SKIP; + } + ksft_finished(); +} diff --git a/tools/testing/selftests/sched_ext/Makefile b/tools/testing/selftests/sched_ext/Makefile index 3cfe90e0f34f..4e06d0baaeec 100644 --- a/tools/testing/selftests/sched_ext/Makefile +++ b/tools/testing/selftests/sched_ext/Makefile @@ -164,10 +164,12 @@ all_test_bpfprogs := $(foreach prog,$(wildcard *.bpf.c),$(INCLUDE_DIR)/$(patsubs auto-test-targets := \ create_dsq \ dequeue \ + dequeue_iter \ enq_last_no_enq_fails \ ddsp_bogus_dsq_fail \ ddsp_vtimelocal_fail \ dsp_local_on \ + enable_cmask \ enq_select_cpu \ exit \ hotplug \ diff --git a/tools/testing/selftests/sched_ext/dequeue_iter.bpf.c b/tools/testing/selftests/sched_ext/dequeue_iter.bpf.c new file mode 100644 index 000000000000..76c2c71a90b6 --- /dev/null +++ b/tools/testing/selftests/sched_ext/dequeue_iter.bpf.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ops.dequeue() of this scheduler iterates the user DSQ it consumes + * tasks from with bpf_iter_scx_dsq, which takes the DSQ lock. + * On a kernel that still runs ops.dequeue() with that lock held, the + * iteration self-deadlocks the CPU - this test wedges the system on + * unfixed kernels instead of failing cleanly. + * + * Copyright (c) 2026 fangqiurong + */ + +#include + +char _license[] SEC("license") = "GPL"; + +UEI_DEFINE(uei); + +#define TEST_DSQ_ID 1000 + +u64 dq_count; + +s32 BPF_STRUCT_OPS_SLEEPABLE(dequeue_iter_init) +{ + return scx_bpf_create_dsq(TEST_DSQ_ID, -1); +} + +s32 BPF_STRUCT_OPS(dequeue_iter_select_cpu, struct task_struct *p, + s32 prev_cpu, u64 wake_flags) +{ + return prev_cpu; +} + +void BPF_STRUCT_OPS(dequeue_iter_enqueue, struct task_struct *p, u64 enq_flags) +{ + scx_bpf_dsq_insert(p, TEST_DSQ_ID, SCX_SLICE_DFL, enq_flags); +} + +void BPF_STRUCT_OPS(dequeue_iter_dispatch, s32 cpu, struct task_struct *task) +{ + scx_bpf_dsq_move_to_local(TEST_DSQ_ID, 0); +} + +void BPF_STRUCT_OPS(dequeue_iter_dequeue, struct task_struct *p, u64 deq_flags) +{ + struct bpf_iter_scx_dsq it; + struct task_struct *t; + + if (!bpf_iter_scx_dsq_new(&it, TEST_DSQ_ID, 0)) { + while ((t = bpf_iter_scx_dsq_next(&it))) + ; + } + bpf_iter_scx_dsq_destroy(&it); + + __sync_fetch_and_add(&dq_count, 1); +} + +void BPF_STRUCT_OPS(dequeue_iter_exit, struct scx_exit_info *ei) +{ + UEI_RECORD(uei, ei); + scx_bpf_destroy_dsq(TEST_DSQ_ID); +} + +SEC(".struct_ops.link") +struct sched_ext_ops dequeue_iter_ops = { + .init = (void *)dequeue_iter_init, + .select_cpu = (void *)dequeue_iter_select_cpu, + .enqueue = (void *)dequeue_iter_enqueue, + .dispatch = (void *)dequeue_iter_dispatch, + .dequeue = (void *)dequeue_iter_dequeue, + .exit = (void *)dequeue_iter_exit, + .timeout_ms = 1000U, + .name = "dequeue_iter", +}; diff --git a/tools/testing/selftests/sched_ext/dequeue_iter.c b/tools/testing/selftests/sched_ext/dequeue_iter.c new file mode 100644 index 000000000000..f60711bf3b39 --- /dev/null +++ b/tools/testing/selftests/sched_ext/dequeue_iter.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 fangqiurong + */ +#include +#include +#include +#include +#include "dequeue_iter.bpf.skel.h" +#include "scx_test.h" + +#define DQ_TARGET 10 +#define DQ_DEADLINE_MS 3000 + +static unsigned long long now_ms(void) +{ + struct timespec ts; + + clock_gettime(CLOCK_MONOTONIC, &ts); + + return ts.tv_sec * 1000ULL + ts.tv_nsec / 1000000; +} + +static enum scx_test_status setup(void **ctx) +{ + struct dequeue_iter *skel; + + skel = dequeue_iter__open(); + SCX_FAIL_IF(!skel, "Failed to open"); + SCX_ENUM_INIT(skel); + SCX_FAIL_IF(dequeue_iter__load(skel), "Failed to load skel"); + + *ctx = skel; + + return SCX_TEST_PASS; +} + +static enum scx_test_status run(void *ctx) +{ + struct dequeue_iter *skel = ctx; + struct bpf_link *link; + unsigned long long end; + + link = bpf_map__attach_struct_ops(skel->maps.dequeue_iter_ops); + SCX_FAIL_IF(!link, "Failed to attach scheduler"); + + end = now_ms() + DQ_DEADLINE_MS; + while (skel->bss->dq_count < DQ_TARGET && !UEI_EXITED(skel, uei) && + now_ms() < end) + usleep(100); + + bpf_link__destroy(link); + + SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_UNREG)); + + if (skel->bss->dq_count < DQ_TARGET) { + SCX_ERR("ops.dequeue() fired only %llu times", + (unsigned long long)skel->bss->dq_count); + return SCX_TEST_FAIL; + } + + return SCX_TEST_PASS; +} + +static void cleanup(void *ctx) +{ + struct dequeue_iter *skel = ctx; + + dequeue_iter__destroy(skel); +} + +struct scx_test dequeue_iter = { + .name = "dequeue_iter", + .description = "Verify ops.dequeue() can iterate its source user DSQ", + .setup = setup, + .run = run, + .cleanup = cleanup, +}; +REGISTER_SCX_TEST(&dequeue_iter) diff --git a/tools/testing/selftests/sched_ext/enable_cmask.bpf.c b/tools/testing/selftests/sched_ext/enable_cmask.bpf.c new file mode 100644 index 000000000000..0068f3c7ab3c --- /dev/null +++ b/tools/testing/selftests/sched_ext/enable_cmask.bpf.c @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * A cid-form scheduler checking the cmask cid-form ops.enable() receives: the + * header, every cid bit against p->cpus_ptr, and that set_cmask() follows with + * the same mask before set_weight() and before the task first becomes runnable, + * and never runs before enable(). + * + * Copyright (c) 2026 Tejun Heo + */ +#include + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1 << 16); +} arena SEC(".maps"); + +struct task_ctx { + u64 enable_fp; /* fingerprint of the mask enable() received */ + bool enabled; + bool pending; /* enable() ran, the initial set_cmask() hasn't */ +}; + +struct { + __uint(type, BPF_MAP_TYPE_TASK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, struct task_ctx); +} task_ctx_stor SEC(".maps"); + +/* details of a cid bit mismatch, filled by check_mask() */ +struct mask_mismatch { + s32 cid; + bool want; + bool got; +}; + +u64 nr_enable, nr_initial_set_cmask, nr_set_cmask, nr_set_weight; + +UEI_DEFINE(uei); + +static struct task_ctx *lookup_task_ctx(struct task_struct *p) +{ + struct task_ctx *tctx; + + tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); + if (!tctx) + scx_bpf_error("task_ctx lookup failed for %s[%d]", p->comm, p->pid); + return tctx; +} + +/* + * Verify @m's header and every cid bit against @p's cpumask and fingerprint the + * bits into @fp. Return 0 on success, -EINVAL on a bad header, -ENOENT on a cid + * without a cpu and -EIO on a bit mismatch with the details in @mm. + */ +static int check_mask(struct task_struct *p, const struct scx_cmask __arena *m, u64 *fp, + struct mask_mismatch *mm) +{ + u32 nr_cids = scx_bpf_nr_cids(); + u64 h = 0; + s32 cid; + + if (m->base || m->nr_cids != nr_cids || m->alloc_words != CMASK_NR_WORDS(nr_cids)) + return -EINVAL; + + bpf_for(cid, 0, nr_cids) { + bool want, got; + s32 cpu; + + cpu = scx_bpf_cid_to_cpu(cid); + if (cpu < 0) + return -ENOENT; + want = bpf_cpumask_test_cpu(cpu, p->cpus_ptr); + got = cmask_test(cid, m); + if (want != got) { + mm->cid = cid; + mm->want = want; + mm->got = got; + return -EIO; + } + h = h * 31 + got; + } + + *fp = h; + return 0; +} + +s32 BPF_STRUCT_OPS_SLEEPABLE(enable_cmask_init_task, struct task_struct *p, + struct scx_init_task_args *args) +{ + if (!bpf_task_storage_get(&task_ctx_stor, p, 0, BPF_LOCAL_STORAGE_GET_F_CREATE)) + return -ENOMEM; + return 0; +} + +void BPF_STRUCT_OPS(enable_cmask_enable, struct task_struct *p, struct scx_enable_args *args) +{ + struct scx_cmask __arena *m = (struct scx_cmask __arena *)args->cmask_arena_addr; + struct mask_mismatch mm = {}; + struct task_ctx *tctx; + int ret; + + asm volatile("" :: "r"(&arena)); + tctx = lookup_task_ctx(p); + if (!tctx) + return; + + __sync_fetch_and_add(&nr_enable, 1); + if (tctx->enabled || tctx->pending) { + scx_bpf_error("enable: %s[%d] enabled twice", p->comm, p->pid); + return; + } + + ret = check_mask(p, m, &tctx->enable_fp, &mm); + if (ret) { + scx_bpf_error("enable: %s[%d] cmask check failed %d cid=%d want=%d got=%d", + p->comm, p->pid, ret, mm.cid, mm.want, mm.got); + return; + } + tctx->enabled = true; + tctx->pending = true; +} + +void BPF_STRUCT_OPS(enable_cmask_set_cmask, struct task_struct *p, + struct scx_cmask __arena *m) +{ + struct mask_mismatch mm = {}; + struct task_ctx *tctx; + u64 fp; + int ret; + + asm volatile("" :: "r"(&arena)); + tctx = lookup_task_ctx(p); + if (!tctx) + return; + + __sync_fetch_and_add(&nr_set_cmask, 1); + if (!tctx->enabled) { + scx_bpf_error("set_cmask: %s[%d] not enabled", p->comm, p->pid); + return; + } + + ret = check_mask(p, m, &fp, &mm); + if (ret) { + scx_bpf_error("set_cmask: %s[%d] cmask check failed %d cid=%d want=%d got=%d", + p->comm, p->pid, ret, mm.cid, mm.want, mm.got); + return; + } + + if (tctx->pending) { + if (fp != tctx->enable_fp) { + scx_bpf_error("set_cmask: %s[%d] initial mask differs from enable()", + p->comm, p->pid); + return; + } + tctx->pending = false; + __sync_fetch_and_add(&nr_initial_set_cmask, 1); + } +} + +void BPF_STRUCT_OPS(enable_cmask_set_weight, struct task_struct *p, u32 weight) +{ + struct task_ctx *tctx; + + tctx = lookup_task_ctx(p); + if (!tctx) + return; + + __sync_fetch_and_add(&nr_set_weight, 1); + if (tctx->pending) + scx_bpf_error("set_weight: %s[%d] before the initial set_cmask()", p->comm, + p->pid); +} + +void BPF_STRUCT_OPS(enable_cmask_runnable, struct task_struct *p, u64 enq_flags) +{ + struct task_ctx *tctx; + + tctx = lookup_task_ctx(p); + if (!tctx) + return; + + if (tctx->pending) + scx_bpf_error("runnable: %s[%d] before the initial set_cmask()", p->comm, + p->pid); +} + +void BPF_STRUCT_OPS(enable_cmask_disable, struct task_struct *p) +{ + struct task_ctx *tctx; + + tctx = lookup_task_ctx(p); + if (!tctx) + return; + + tctx->enabled = false; + tctx->pending = false; +} + +void BPF_STRUCT_OPS(enable_cmask_exit, struct scx_exit_info *ei) +{ + UEI_RECORD(uei, ei); +} + +SCX_OPS_CID_DEFINE(enable_cmask_ops, + .init_task = (void *)enable_cmask_init_task, + .enable = (void *)enable_cmask_enable, + .set_cmask = (void *)enable_cmask_set_cmask, + .set_weight = (void *)enable_cmask_set_weight, + .runnable = (void *)enable_cmask_runnable, + .disable = (void *)enable_cmask_disable, + .exit = (void *)enable_cmask_exit, + .flags = SCX_OPS_SWITCH_PARTIAL, + .name = "enable_cmask"); diff --git a/tools/testing/selftests/sched_ext/enable_cmask.c b/tools/testing/selftests/sched_ext/enable_cmask.c new file mode 100644 index 000000000000..556bcad4431d --- /dev/null +++ b/tools/testing/selftests/sched_ext/enable_cmask.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Tejun Heo */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include "enable_cmask.bpf.skel.h" +#include "scx_test.h" + +#define SCHED_EXT 7 +#define NR_CHILDREN 8 +#define MAX_CPUS 1024 + +static int cpus[MAX_CPUS]; +static int nr_cpus; + +static void spin_ms(int ms) +{ + struct timespec start, now; + + clock_gettime(CLOCK_MONOTONIC, &start); + do { + clock_gettime(CLOCK_MONOTONIC, &now); + } while ((now.tv_sec - start.tv_sec) * 1000 + + (now.tv_nsec - start.tv_nsec) / 1000000 < ms); +} + +static int pin(pid_t pid, int idx) +{ + cpu_set_t set; + + CPU_ZERO(&set); + CPU_SET(cpus[idx % nr_cpus], &set); + return sched_setaffinity(pid, sizeof(set), &set); +} + +/* + * Pin, switch to SCHED_EXT for a class-switch enable, fork a grandchild that + * inherits the policy for a fork-path enable, then change affinity a few times + * while running for set_cmask() on live tasks. + */ +static int child(int idx) +{ + struct sched_param param = {}; + int i, status; + pid_t pid; + + if (pin(0, idx) || sched_setscheduler(0, SCHED_EXT, ¶m)) + return 1; + + pid = fork(); + if (pid < 0) + return 1; + if (!pid) { + spin_ms(20); + return 0; + } + + for (i = 1; i <= 4; i++) { + if (pin(0, idx + i)) + return 1; + spin_ms(5); + } + + return waitpid(pid, &status, 0) == pid && !status ? 0 : 1; +} + +static enum scx_test_status run(void *ctx) +{ + struct enable_cmask *skel; + struct bpf_link *link; + pid_t pids[NR_CHILDREN]; + cpu_set_t set; + int i, status, failed = 0; + + if (!__COMPAT_struct_has_field("scx_enable_args", "cmask_arena_addr")) + return SCX_TEST_SKIP; + + SCX_FAIL_IF(sched_getaffinity(0, sizeof(set), &set), "Failed to read affinity"); + for (i = 0; i < MAX_CPUS && i < CPU_SETSIZE; i++) + if (CPU_ISSET(i, &set)) + cpus[nr_cpus++] = i; + if (nr_cpus < 2) + return SCX_TEST_SKIP; + + skel = enable_cmask__open(); + SCX_FAIL_IF(!skel, "Failed to open"); + SCX_ENUM_INIT(skel); + SCX_FAIL_IF(enable_cmask__load(skel), "Failed to load skel"); + + link = bpf_map__attach_struct_ops(skel->maps.enable_cmask_ops); + SCX_FAIL_IF(!link, "Failed to attach struct_ops"); + + for (i = 0; i < NR_CHILDREN; i++) { + pids[i] = fork(); + SCX_FAIL_IF(pids[i] < 0, "Failed to fork"); + if (!pids[i]) + exit(child(i)); + } + + /* affinity changes from the outside race with the children's own */ + for (i = 0; i < NR_CHILDREN; i++) + pin(pids[i], i + NR_CHILDREN); + + for (i = 0; i < NR_CHILDREN; i++) { + if (waitpid(pids[i], &status, 0) != pids[i] || status) + failed++; + } + + bpf_link__destroy(link); + + SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_UNREG)); + SCX_EQ(failed, 0); + SCX_GE(skel->bss->nr_enable, 2 * NR_CHILDREN); + SCX_EQ(skel->bss->nr_initial_set_cmask, skel->bss->nr_enable); + SCX_GT(skel->bss->nr_set_cmask, skel->bss->nr_initial_set_cmask); + SCX_GE(skel->bss->nr_set_weight, skel->bss->nr_enable); + printf("enable=%lu initial_set_cmask=%lu set_cmask=%lu set_weight=%lu\n", + (unsigned long)skel->bss->nr_enable, + (unsigned long)skel->bss->nr_initial_set_cmask, + (unsigned long)skel->bss->nr_set_cmask, + (unsigned long)skel->bss->nr_set_weight); + + enable_cmask__destroy(skel); + return SCX_TEST_PASS; +} + +struct scx_test enable_cmask = { + .name = "enable_cmask", + .description = "Check the cid-form ops.enable() cmask and the set_cmask() after it", + .run = run, +}; +REGISTER_SCX_TEST(&enable_cmask) diff --git a/tools/testing/selftests/tc-testing/tc-tests/filters/u32.json b/tools/testing/selftests/tc-testing/tc-tests/filters/u32.json index e2b03f2b5e89..edc5148a8d97 100644 --- a/tools/testing/selftests/tc-testing/tc-tests/filters/u32.json +++ b/tools/testing/selftests/tc-testing/tc-tests/filters/u32.json @@ -376,5 +376,53 @@ "teardown": [ "$TC qdisc del dev $DUMMY clsact" ] + }, + { + "id": "35fc", + "name": "u32 manual table then auto table: auto allocation must not alias a live manual handle", + "category": [ + "filter", + "u32" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress", + "$TC filter add dev $DEV1 ingress protocol ip pref 1 handle 801: u32 divisor 16" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 ingress protocol ip pref 2 u32 divisor 16", + "expExitCode": "0", + "verifyCmd": "$TC -d filter show dev $DEV1 ingress", + "matchPattern": "fh 801:", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "a6e8", + "name": "u32 manual table add/del does not leak its idr entry (re-adding the same handle succeeds)", + "category": [ + "filter", + "u32" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress", + "$TC filter add dev $DEV1 ingress protocol ip pref 1 u32 divisor 16", + "$TC filter add dev $DEV1 ingress protocol ip pref 5 handle 901: u32 divisor 1", + "$TC filter del dev $DEV1 ingress protocol ip pref 5 handle 901: u32" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 ingress protocol ip pref 6 handle 901: u32 divisor 1", + "expExitCode": "0", + "verifyCmd": "$TC -d filter show dev $DEV1 ingress", + "matchPattern": "fh 901: ht divisor 1", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] } ] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/hfsc.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/hfsc.json index c98c339424d4..4f6bbb8b57f9 100644 --- a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/hfsc.json +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/hfsc.json @@ -169,5 +169,39 @@ "teardown": [ "$TC qdisc del dev $DUMMY handle 1: root" ] + }, + { + "id": "8c39", + "name": "HFSC classify walk still reaches leaf after lateral drift", + "category": [ + "qdisc", + "hfsc" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "ip link set lo up", + "$TC qdisc add dev lo handle 1: root hfsc default 30", + "$TC class add dev lo parent 1: classid 1:1 hfsc rt m2 100kbit", + "$TC class add dev lo parent 1:1 classid 1:10 hfsc rt m2 50kbit", + "$TC class add dev lo parent 1: classid 1:2 hfsc rt m2 100kbit", + "$TC filter add dev lo parent 1: protocol ip prio 1 u32 match u8 0 0 at 0 flowid 1:1", + "$TC filter add dev lo parent 1:1 protocol ip prio 1 u32 match u8 0 0 at 0 flowid 1:2", + "$TC class add dev lo parent 1:2 classid 1:20 hfsc rt m2 10kbit", + "$TC class add dev lo parent 1: classid 1:3 hfsc rt m2 100kbit", + "$TC filter add dev lo parent 1:2 protocol ip prio 1 u32 match u8 0 0 at 0 flowid 1:3", + "$TC class add dev lo parent 1:3 classid 1:30 hfsc rt m2 10kbit", + "$TC class add dev lo parent 1:3 classid 1:31 hfsc rt m2 100kbit", + "$TC filter add dev lo parent 1:3 protocol ip prio 1 u32 match u8 0 0 at 0 flowid 1:31" + ], + "cmdUnderTest": "ping -n -c 10 -W 1 127.0.0.1", + "expExitCode": "0", + "verifyCmd": "$TC -s class show dev lo", + "matchPattern": "class hfsc 1:31 parent 1:3 rt[^\\n]*\\n Sent [0-9]+ bytes [1-9][0-9]* pkt", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev lo handle 1: root" + ] } ] diff --git a/tools/testing/selftests/ublk/kublk.c b/tools/testing/selftests/ublk/kublk.c index 2400b4615766..5b19bfcfa8d7 100644 --- a/tools/testing/selftests/ublk/kublk.c +++ b/tools/testing/selftests/ublk/kublk.c @@ -1589,7 +1589,7 @@ static int ublk_start_daemon(const struct dev_ctx *ctx, struct ublk_dev *dev) /* Signal listener thread to stop and wait for it */ if (linfo.stop_efd >= 0) { - write(linfo.stop_efd, &stop_val, sizeof(stop_val)); + (void)!write(linfo.stop_efd, &stop_val, sizeof(stop_val)); pthread_join(listener, NULL); close(linfo.stop_efd); ublk_shmem_sock_destroy(dinfo->dev_id, linfo.sock_fd); diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 65eb26a0520d..5a782849521c 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1117,7 +1117,7 @@ static struct kvm *kvm_create_vm(unsigned long type, const char *fdname) rcuwait_init(&kvm->mn_memslots_update_rcuwait); xa_init(&kvm->vcpu_array); #ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES - xa_init(&kvm->mem_attr_array); + xa_init_flags(&kvm->mem_attr_array, XA_FLAGS_ACCOUNT); #endif INIT_LIST_HEAD(&kvm->gpc_list); @@ -1363,6 +1363,9 @@ int kvm_trylock_all_vcpus(struct kvm *kvm) lockdep_assert_held(&kvm->lock); + if (WARN_ON_ONCE(kvm_is_vcpu_creation_in_progress(kvm))) + return -EBUSY; + kvm_for_each_vcpu(i, vcpu, kvm) if (!mutex_trylock_nest_lock(&vcpu->mutex, &kvm->lock)) goto out_unlock; @@ -1386,6 +1389,9 @@ int kvm_lock_all_vcpus(struct kvm *kvm) lockdep_assert_held(&kvm->lock); + if (WARN_ON_ONCE(kvm_is_vcpu_creation_in_progress(kvm))) + return -EBUSY; + kvm_for_each_vcpu(i, vcpu, kvm) { r = mutex_lock_killable_nest_lock(&vcpu->mutex, &kvm->lock); if (r) @@ -2447,14 +2453,36 @@ bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end, return (kvm_get_memory_attributes(kvm, start) & mask) == attrs; guard(rcu)(); - if (!attrs) - return !xas_find(&xas, end - 1); + /* + * Lookup the entry for each index instead of iterating over the xarray + * as KVM deletes/nullifies entries to represent "no attributes", and + * the xas index is effectively invalid when no entry is found. I.e. + * matching non-zero attributes for *every* entry effectively requires + * a manually lookup for each index. + * + * Skip pre-allocated, reserved entries, or restart the lookup if the + * xarray was concurrently modified, via xas_retry() ("retry" means the + * entry holds an internal xarray value, i.e. is either invalid or NULL + * from the caller's perspective). + * + * Use xas_next() when looking for non-zero attributes to optimize for + * the case where the start of the range (or the entire range) doesn't + * have any attributes, as xas_next() returns literally the next entry, + * whereas xas_next_entry() returns the next non-NULL entry (bounded by + * a maximum index). + */ for (index = start; index < end; index++) { do { - entry = xas_next(&xas); + entry = attrs ? xas_next(&xas) : + xas_next_entry(&xas, end - 1); } while (xas_retry(&xas, entry)); + if (!entry) + return !attrs; + + WARN_ON_ONCE(!xa_to_value(entry)); + if (xas.xa_index != index || (xa_to_value(entry) & mask) != attrs) return false; @@ -2571,9 +2599,10 @@ static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end, /* * Reserve memory ahead of time to avoid having to deal with failures - * partway through setting the new attributes. + * partway through setting the new attributes. Storing NULL never + * allocates, so no reservations are needed when clearing. */ - for (i = start; i < end; i++) { + for (i = start; entry && i < end; i++) { r = xa_reserve(&kvm->mem_attr_array, i, GFP_KERNEL_ACCOUNT); if (r) goto out_unlock; @@ -4159,6 +4188,8 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id) struct kvm_vcpu *vcpu; struct page *page; + guard(mutex)(&kvm->lock); + /* * KVM tracks vCPU IDs as 'int', be kind to userspace and reject * too-large values instead of silently truncating. @@ -4171,26 +4202,17 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id) if (id >= KVM_MAX_VCPU_IDS) return -EINVAL; - mutex_lock(&kvm->lock); - if (kvm->created_vcpus >= kvm->max_vcpus) { - mutex_unlock(&kvm->lock); + if (kvm->created_vcpus >= kvm->max_vcpus) return -EINVAL; - } - if (test_bit(id, kvm->vcpu_ids)) { - mutex_unlock(&kvm->lock); + if (kvm_get_vcpu_by_id(kvm, id)) return -EEXIST; - } r = kvm_arch_vcpu_precreate(kvm, id); - if (r) { - mutex_unlock(&kvm->lock); + if (r) return r; - } kvm->created_vcpus++; - __set_bit(id, kvm->vcpu_ids); - mutex_unlock(&kvm->lock); vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL_ACCOUNT); if (!vcpu) { @@ -4221,13 +4243,6 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id) goto arch_vcpu_destroy; } - mutex_lock(&kvm->lock); - - if (WARN_ON_ONCE(kvm_get_vcpu_by_id(kvm, id))) { - r = -EEXIST; - goto unlock_vcpu_destroy; - } - /* * Set the vCPU's index *before* the vCPU is reachable by other tasks. * Unwind the index back to -1 on failure so that KVM can use the index @@ -4261,7 +4276,6 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id) atomic_inc(&kvm->online_vcpus); mutex_unlock(&vcpu->mutex); - mutex_unlock(&kvm->lock); kvm_arch_vcpu_postcreate(vcpu); kvm_create_vcpu_debugfs(vcpu); return r; @@ -4272,7 +4286,6 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id) xa_erase(&kvm->vcpu_array, vcpu->vcpu_idx); unlock_vcpu_destroy: vcpu->vcpu_idx = -1; - mutex_unlock(&kvm->lock); kvm_dirty_ring_free(&vcpu->dirty_ring); arch_vcpu_destroy: kvm_arch_vcpu_destroy(vcpu); @@ -4281,10 +4294,7 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id) vcpu_free: kmem_cache_free(kvm_vcpu_cache, vcpu); vcpu_decrement: - mutex_lock(&kvm->lock); kvm->created_vcpus--; - __clear_bit(id, kvm->vcpu_ids); - mutex_unlock(&kvm->lock); return r; }